2024-11-10 09:24:57,094 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 09:24:57,107 main DEBUG Took 0.010973 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 09:24:57,108 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 09:24:57,108 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 09:24:57,109 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 09:24:57,110 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,117 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 09:24:57,129 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,131 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,132 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,132 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,134 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,135 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,136 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,136 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,137 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,138 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,138 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,139 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,139 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,140 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,141 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,141 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 09:24:57,142 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,142 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 09:24:57,144 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 09:24:57,145 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 09:24:57,147 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 09:24:57,147 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 09:24:57,149 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 09:24:57,149 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 09:24:57,158 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 09:24:57,160 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 09:24:57,162 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 09:24:57,163 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 09:24:57,163 main DEBUG createAppenders(={Console}) 2024-11-10 09:24:57,164 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 09:24:57,164 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 09:24:57,165 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 09:24:57,165 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 09:24:57,165 main DEBUG OutputStream closed 2024-11-10 09:24:57,166 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 09:24:57,166 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 09:24:57,166 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 09:24:57,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 09:24:57,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 09:24:57,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 09:24:57,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 09:24:57,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 09:24:57,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 09:24:57,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 09:24:57,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 09:24:57,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 09:24:57,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 09:24:57,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 09:24:57,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 09:24:57,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 09:24:57,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 09:24:57,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 09:24:57,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 09:24:57,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 09:24:57,258 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 09:24:57,260 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 09:24:57,261 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 09:24:57,261 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 09:24:57,262 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T09:24:57,481 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5 2024-11-10 09:24:57,484 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 09:24:57,484 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T09:24:57,492 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-10T09:24:57,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T09:24:57,529 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f, deleteOnExit=true 2024-11-10T09:24:57,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T09:24:57,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/test.cache.data in system properties and HBase conf 2024-11-10T09:24:57,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T09:24:57,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/hadoop.log.dir in system properties and HBase conf 2024-11-10T09:24:57,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T09:24:57,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T09:24:57,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T09:24:57,628 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T09:24:57,730 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T09:24:57,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T09:24:57,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T09:24:57,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T09:24:57,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T09:24:57,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T09:24:57,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T09:24:57,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T09:24:57,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T09:24:57,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T09:24:57,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/nfs.dump.dir in system properties and HBase conf 2024-11-10T09:24:57,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/java.io.tmpdir in system properties and HBase conf 2024-11-10T09:24:57,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T09:24:57,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T09:24:57,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T09:24:58,946 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T09:24:59,018 INFO [Time-limited test {}] log.Log(170): Logging initialized @2758ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T09:24:59,091 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T09:24:59,148 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T09:24:59,172 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T09:24:59,172 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T09:24:59,173 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T09:24:59,186 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T09:24:59,189 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/hadoop.log.dir/,AVAILABLE} 2024-11-10T09:24:59,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T09:24:59,419 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/java.io.tmpdir/jetty-localhost-36863-hadoop-hdfs-3_4_1-tests_jar-_-any-2565324497806145069/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T09:24:59,426 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:36863} 2024-11-10T09:24:59,426 INFO [Time-limited test {}] server.Server(415): Started @3167ms 2024-11-10T09:24:59,992 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T09:25:00,000 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T09:25:00,003 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T09:25:00,003 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T09:25:00,003 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T09:25:00,004 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/hadoop.log.dir/,AVAILABLE} 2024-11-10T09:25:00,005 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T09:25:00,103 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/java.io.tmpdir/jetty-localhost-35937-hadoop-hdfs-3_4_1-tests_jar-_-any-13706447929611928348/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T09:25:00,104 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:35937} 2024-11-10T09:25:00,104 INFO [Time-limited test {}] server.Server(415): Started @3844ms 2024-11-10T09:25:00,156 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T09:25:01,926 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/data/data2/current/BP-1012386263-172.17.0.3-1731230698324/current, will proceed with Du for space computation calculation, 2024-11-10T09:25:01,927 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/data/data1/current/BP-1012386263-172.17.0.3-1731230698324/current, will proceed with Du for space computation calculation, 2024-11-10T09:25:01,990 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T09:25:02,055 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7d6da296a4e9a82c with lease ID 0xac043f4ad799a438: Processing first storage report for DS-382c9e5c-d785-4895-9a61-8c7ab6cd67c7 from datanode DatanodeRegistration(127.0.0.1:32877, datanodeUuid=23b75091-1942-4c7e-b8ea-bd802bc427a3, infoPort=46727, infoSecurePort=0, ipcPort=44115, storageInfo=lv=-57;cid=testClusterID;nsid=1517600920;c=1731230698324) 2024-11-10T09:25:02,056 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7d6da296a4e9a82c with lease ID 0xac043f4ad799a438: from storage DS-382c9e5c-d785-4895-9a61-8c7ab6cd67c7 node DatanodeRegistration(127.0.0.1:32877, datanodeUuid=23b75091-1942-4c7e-b8ea-bd802bc427a3, infoPort=46727, infoSecurePort=0, ipcPort=44115, storageInfo=lv=-57;cid=testClusterID;nsid=1517600920;c=1731230698324), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T09:25:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7d6da296a4e9a82c with lease ID 0xac043f4ad799a438: Processing first storage report for DS-1c6f7cb4-6db2-4317-b3f7-1bdb64ffc829 from datanode DatanodeRegistration(127.0.0.1:32877, datanodeUuid=23b75091-1942-4c7e-b8ea-bd802bc427a3, infoPort=46727, infoSecurePort=0, ipcPort=44115, storageInfo=lv=-57;cid=testClusterID;nsid=1517600920;c=1731230698324) 2024-11-10T09:25:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7d6da296a4e9a82c with lease ID 0xac043f4ad799a438: from storage DS-1c6f7cb4-6db2-4317-b3f7-1bdb64ffc829 node DatanodeRegistration(127.0.0.1:32877, datanodeUuid=23b75091-1942-4c7e-b8ea-bd802bc427a3, infoPort=46727, infoSecurePort=0, ipcPort=44115, storageInfo=lv=-57;cid=testClusterID;nsid=1517600920;c=1731230698324), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T09:25:02,151 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5 2024-11-10T09:25:02,291 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/zookeeper_0, clientPort=57057, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T09:25:02,319 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57057 2024-11-10T09:25:02,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:25:02,340 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:25:02,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741825_1001 (size=7) 2024-11-10T09:25:03,086 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 with version=8 2024-11-10T09:25:03,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/hbase-staging 2024-11-10T09:25:03,205 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T09:25:03,475 INFO [Time-limited test {}] client.ConnectionUtils(128): master/802346a334f5:0 server-side Connection retries=45 2024-11-10T09:25:03,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:25:03,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T09:25:03,496 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T09:25:03,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:25:03,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T09:25:03,701 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T09:25:03,782 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T09:25:03,793 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T09:25:03,798 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T09:25:03,846 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 34603 (auto-detected) 2024-11-10T09:25:03,847 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-10T09:25:03,879 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38235 2024-11-10T09:25:03,929 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38235 connecting to ZooKeeper ensemble=127.0.0.1:57057 2024-11-10T09:25:04,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:382350x0, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T09:25:04,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38235-0x101241016650000 connected 2024-11-10T09:25:04,324 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:25:04,326 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:25:04,334 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:25:04,337 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6, hbase.cluster.distributed=false 2024-11-10T09:25:04,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T09:25:04,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38235 2024-11-10T09:25:04,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38235 2024-11-10T09:25:04,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38235 2024-11-10T09:25:04,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38235 2024-11-10T09:25:04,366 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38235 2024-11-10T09:25:04,475 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/802346a334f5:0 server-side Connection retries=45 2024-11-10T09:25:04,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:25:04,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T09:25:04,478 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T09:25:04,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T09:25:04,479 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T09:25:04,482 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T09:25:04,485 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T09:25:04,486 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32883 2024-11-10T09:25:04,488 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32883 connecting to ZooKeeper ensemble=127.0.0.1:57057 2024-11-10T09:25:04,490 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:25:04,493 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:25:04,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328830x0, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T09:25:04,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328830x0, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:25:04,510 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32883-0x101241016650001 connected 2024-11-10T09:25:04,513 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T09:25:04,521 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T09:25:04,524 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T09:25:04,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T09:25:04,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32883 2024-11-10T09:25:04,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32883 2024-11-10T09:25:04,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32883 2024-11-10T09:25:04,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32883 2024-11-10T09:25:04,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32883 2024-11-10T09:25:04,554 DEBUG [M:0;802346a334f5:38235 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;802346a334f5:38235 2024-11-10T09:25:04,555 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/802346a334f5,38235,1731230703274 2024-11-10T09:25:04,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:25:04,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:25:04,573 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/802346a334f5,38235,1731230703274 2024-11-10T09:25:04,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T09:25:04,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:04,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:04,598 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T09:25:04,599 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/802346a334f5,38235,1731230703274 from backup master directory 2024-11-10T09:25:04,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:25:04,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/802346a334f5,38235,1731230703274 2024-11-10T09:25:04,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T09:25:04,608 WARN [master/802346a334f5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T09:25:04,608 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=802346a334f5,38235,1731230703274 2024-11-10T09:25:04,610 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T09:25:04,611 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T09:25:04,661 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/hbase.id] with ID: 31bb2194-f5a5-4093-a2ad-d78528cf56b8 2024-11-10T09:25:04,661 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/.tmp/hbase.id 2024-11-10T09:25:04,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741826_1002 (size=42) 2024-11-10T09:25:05,073 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/.tmp/hbase.id]:[hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/hbase.id] 2024-11-10T09:25:05,122 INFO [master/802346a334f5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T09:25:05,126 INFO [master/802346a334f5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T09:25:05,145 INFO [master/802346a334f5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-10T09:25:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:05,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:05,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741827_1003 (size=196) 2024-11-10T09:25:05,244 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:25:05,247 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T09:25:05,263 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:05,269 INFO [master/802346a334f5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T09:25:05,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741828_1004 (size=1189) 2024-11-10T09:25:05,726 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store 2024-11-10T09:25:05,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741829_1005 (size=34) 2024-11-10T09:25:06,150 INFO [master/802346a334f5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T09:25:06,153 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:06,154 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T09:25:06,155 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:25:06,155 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:25:06,156 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T09:25:06,156 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:25:06,157 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:25:06,158 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731230706154Disabling compacts and flushes for region at 1731230706154Disabling writes for close at 1731230706156 (+2 ms)Writing region close event to WAL at 1731230706157 (+1 ms)Closed at 1731230706157 2024-11-10T09:25:06,160 WARN [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/.initializing 2024-11-10T09:25:06,160 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/WALs/802346a334f5,38235,1731230703274 2024-11-10T09:25:06,197 INFO [master/802346a334f5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T09:25:06,214 INFO [master/802346a334f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=802346a334f5%2C38235%2C1731230703274, suffix=, logDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/WALs/802346a334f5,38235,1731230703274, archiveDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/oldWALs, maxLogs=10 2024-11-10T09:25:06,236 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/WALs/802346a334f5,38235,1731230703274/802346a334f5%2C38235%2C1731230703274.1731230706219, exclude list is [], retry=0 2024-11-10T09:25:06,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32877,DS-382c9e5c-d785-4895-9a61-8c7ab6cd67c7,DISK] 2024-11-10T09:25:06,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T09:25:06,287 INFO [master/802346a334f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/WALs/802346a334f5,38235,1731230703274/802346a334f5%2C38235%2C1731230703274.1731230706219 2024-11-10T09:25:06,288 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46727:46727)] 2024-11-10T09:25:06,288 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:25:06,289 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:06,291 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,292 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,328 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T09:25:06,355 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:06,357 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:06,357 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,361 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T09:25:06,361 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:06,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:06,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T09:25:06,365 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:06,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:06,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,369 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T09:25:06,369 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:06,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:06,370 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,374 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,375 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,381 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,382 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,386 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T09:25:06,390 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T09:25:06,394 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:25:06,395 INFO [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61862605, jitterRate=-0.07817535102367401}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T09:25:06,402 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731230706304Initializing all the Stores at 1731230706306 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230706307 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230706307Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230706308 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230706308Cleaning up temporary data from old regions at 1731230706382 (+74 ms)Region opened successfully at 1731230706401 (+19 ms) 2024-11-10T09:25:06,403 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T09:25:06,431 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7f2169, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=802346a334f5/172.17.0.3:0 2024-11-10T09:25:06,457 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T09:25:06,466 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T09:25:06,466 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T09:25:06,468 INFO [master/802346a334f5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T09:25:06,469 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T09:25:06,473 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-10T09:25:06,474 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T09:25:06,496 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T09:25:06,504 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T09:25:06,718 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T09:25:06,727 INFO [master/802346a334f5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T09:25:06,730 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T09:25:06,750 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T09:25:06,753 INFO [master/802346a334f5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T09:25:06,757 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T09:25:06,797 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T09:25:06,800 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T09:25:06,887 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T09:25:06,905 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T09:25:06,962 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T09:25:07,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T09:25:07,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T09:25:07,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:07,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:07,037 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=802346a334f5,38235,1731230703274, sessionid=0x101241016650000, setting cluster-up flag (Was=false) 2024-11-10T09:25:07,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:07,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:07,112 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T09:25:07,115 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=802346a334f5,38235,1731230703274 2024-11-10T09:25:07,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:07,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:07,171 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T09:25:07,173 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=802346a334f5,38235,1731230703274 2024-11-10T09:25:07,181 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T09:25:07,247 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(746): ClusterId : 31bb2194-f5a5-4093-a2ad-d78528cf56b8 2024-11-10T09:25:07,252 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T09:25:07,261 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T09:25:07,268 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T09:25:07,269 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T09:25:07,274 INFO [master/802346a334f5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T09:25:07,282 INFO [master/802346a334f5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T09:25:07,282 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T09:25:07,283 DEBUG [RS:0;802346a334f5:32883 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e79446, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=802346a334f5/172.17.0.3:0 2024-11-10T09:25:07,289 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 802346a334f5,38235,1731230703274 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T09:25:07,297 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:25:07,297 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:25:07,298 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:25:07,298 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/802346a334f5:0, corePoolSize=5, maxPoolSize=5 2024-11-10T09:25:07,299 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/802346a334f5:0, corePoolSize=10, maxPoolSize=10 2024-11-10T09:25:07,299 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,299 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/802346a334f5:0, corePoolSize=2, maxPoolSize=2 2024-11-10T09:25:07,299 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,304 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;802346a334f5:32883 2024-11-10T09:25:07,308 INFO [RS:0;802346a334f5:32883 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T09:25:07,309 INFO [RS:0;802346a334f5:32883 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T09:25:07,309 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T09:25:07,311 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T09:25:07,311 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T09:25:07,313 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(2659): reportForDuty to master=802346a334f5,38235,1731230703274 with port=32883, startcode=1731230704436 2024-11-10T09:25:07,315 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731230737315 2024-11-10T09:25:07,317 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T09:25:07,318 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T09:25:07,319 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:07,320 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T09:25:07,321 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T09:25:07,322 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T09:25:07,322 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T09:25:07,322 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T09:25:07,329 DEBUG [RS:0;802346a334f5:32883 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T09:25:07,327 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,338 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T09:25:07,340 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T09:25:07,342 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T09:25:07,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741831_1007 (size=1321) 2024-11-10T09:25:07,345 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T09:25:07,346 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:25:07,356 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T09:25:07,357 INFO [master/802346a334f5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T09:25:07,364 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.large.0-1731230707358,5,FailOnTimeoutGroup] 2024-11-10T09:25:07,371 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.small.0-1731230707364,5,FailOnTimeoutGroup] 2024-11-10T09:25:07,372 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,372 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T09:25:07,373 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,374 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741832_1008 (size=32) 2024-11-10T09:25:07,393 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T09:25:07,399 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38235 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 802346a334f5,32883,1731230704436 2024-11-10T09:25:07,401 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38235 {}] master.ServerManager(517): Registering regionserver=802346a334f5,32883,1731230704436 2024-11-10T09:25:07,415 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:25:07,415 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37473 2024-11-10T09:25:07,415 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T09:25:07,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T09:25:07,428 DEBUG [RS:0;802346a334f5:32883 {}] zookeeper.ZKUtil(111): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/802346a334f5,32883,1731230704436 2024-11-10T09:25:07,428 WARN [RS:0;802346a334f5:32883 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T09:25:07,429 INFO [RS:0;802346a334f5:32883 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T09:25:07,429 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/WALs/802346a334f5,32883,1731230704436 2024-11-10T09:25:07,431 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [802346a334f5,32883,1731230704436] 2024-11-10T09:25:07,455 INFO [RS:0;802346a334f5:32883 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T09:25:07,476 INFO [RS:0;802346a334f5:32883 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T09:25:07,482 INFO [RS:0;802346a334f5:32883 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T09:25:07,482 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,483 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T09:25:07,488 INFO [RS:0;802346a334f5:32883 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T09:25:07,489 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,489 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/802346a334f5:0, corePoolSize=2, maxPoolSize=2 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,490 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,491 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,491 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,491 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/802346a334f5:0, corePoolSize=1, maxPoolSize=1 2024-11-10T09:25:07,491 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/802346a334f5:0, corePoolSize=3, maxPoolSize=3 2024-11-10T09:25:07,491 DEBUG [RS:0;802346a334f5:32883 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0, corePoolSize=3, maxPoolSize=3 2024-11-10T09:25:07,492 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,493 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,493 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,493 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,493 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,493 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,32883,1731230704436-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T09:25:07,512 INFO [RS:0;802346a334f5:32883 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T09:25:07,514 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,32883,1731230704436-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,514 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,514 INFO [RS:0;802346a334f5:32883 {}] regionserver.Replication(171): 802346a334f5,32883,1731230704436 started 2024-11-10T09:25:07,533 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:07,533 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1482): Serving as 802346a334f5,32883,1731230704436, RpcServer on 802346a334f5/172.17.0.3:32883, sessionid=0x101241016650001 2024-11-10T09:25:07,534 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T09:25:07,534 DEBUG [RS:0;802346a334f5:32883 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 802346a334f5,32883,1731230704436 2024-11-10T09:25:07,534 DEBUG [RS:0;802346a334f5:32883 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '802346a334f5,32883,1731230704436' 2024-11-10T09:25:07,534 DEBUG [RS:0;802346a334f5:32883 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T09:25:07,535 DEBUG [RS:0;802346a334f5:32883 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T09:25:07,536 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T09:25:07,536 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T09:25:07,536 DEBUG [RS:0;802346a334f5:32883 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 802346a334f5,32883,1731230704436 2024-11-10T09:25:07,536 DEBUG [RS:0;802346a334f5:32883 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '802346a334f5,32883,1731230704436' 2024-11-10T09:25:07,536 DEBUG [RS:0;802346a334f5:32883 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T09:25:07,537 DEBUG [RS:0;802346a334f5:32883 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T09:25:07,538 DEBUG [RS:0;802346a334f5:32883 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T09:25:07,538 INFO [RS:0;802346a334f5:32883 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T09:25:07,538 INFO [RS:0;802346a334f5:32883 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T09:25:07,642 INFO [RS:0;802346a334f5:32883 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T09:25:07,646 INFO [RS:0;802346a334f5:32883 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=802346a334f5%2C32883%2C1731230704436, suffix=, logDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/WALs/802346a334f5,32883,1731230704436, archiveDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/oldWALs, maxLogs=32 2024-11-10T09:25:07,663 DEBUG [RS:0;802346a334f5:32883 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/WALs/802346a334f5,32883,1731230704436/802346a334f5%2C32883%2C1731230704436.1731230707649, exclude list is [], retry=0 2024-11-10T09:25:07,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32877,DS-382c9e5c-d785-4895-9a61-8c7ab6cd67c7,DISK] 2024-11-10T09:25:07,670 INFO [RS:0;802346a334f5:32883 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/WALs/802346a334f5,32883,1731230704436/802346a334f5%2C32883%2C1731230704436.1731230707649 2024-11-10T09:25:07,670 DEBUG [RS:0;802346a334f5:32883 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46727:46727)] 2024-11-10T09:25:07,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:07,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T09:25:07,796 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T09:25:07,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:07,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:07,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T09:25:07,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T09:25:07,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:07,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:07,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T09:25:07,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T09:25:07,805 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:07,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:07,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T09:25:07,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T09:25:07,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:07,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:07,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T09:25:07,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740 2024-11-10T09:25:07,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740 2024-11-10T09:25:07,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T09:25:07,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T09:25:07,821 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:25:07,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T09:25:07,829 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:25:07,830 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65420938, jitterRate=-0.0251520574092865}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:25:07,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731230707787Initializing all the Stores at 1731230707791 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230707792 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230707792Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230707792Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230707792Cleaning up temporary data from old regions at 1731230707820 (+28 ms)Region opened successfully at 1731230707834 (+14 ms) 2024-11-10T09:25:07,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T09:25:07,834 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T09:25:07,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T09:25:07,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T09:25:07,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T09:25:07,836 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T09:25:07,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731230707834Disabling compacts and flushes for region at 1731230707834Disabling writes for close at 1731230707835 (+1 ms)Writing region close event to WAL at 1731230707836 (+1 ms)Closed at 1731230707836 2024-11-10T09:25:07,841 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T09:25:07,841 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T09:25:07,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T09:25:07,856 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T09:25:07,859 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T09:25:08,013 DEBUG [802346a334f5:38235 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T09:25:08,026 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:08,033 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 802346a334f5,32883,1731230704436, state=OPENING 2024-11-10T09:25:08,080 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T09:25:08,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:08,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:25:08,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:25:08,092 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:25:08,093 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T09:25:08,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:25:08,274 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T09:25:08,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50633, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T09:25:08,294 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T09:25:08,294 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T09:25:08,295 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T09:25:08,299 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=802346a334f5%2C32883%2C1731230704436.meta, suffix=.meta, logDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/WALs/802346a334f5,32883,1731230704436, archiveDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/oldWALs, maxLogs=32 2024-11-10T09:25:08,320 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/WALs/802346a334f5,32883,1731230704436/802346a334f5%2C32883%2C1731230704436.meta.1731230708301.meta, exclude list is [], retry=0 2024-11-10T09:25:08,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32877,DS-382c9e5c-d785-4895-9a61-8c7ab6cd67c7,DISK] 2024-11-10T09:25:08,329 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/WALs/802346a334f5,32883,1731230704436/802346a334f5%2C32883%2C1731230704436.meta.1731230708301.meta 2024-11-10T09:25:08,329 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46727:46727)] 2024-11-10T09:25:08,330 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:25:08,331 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T09:25:08,334 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T09:25:08,340 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T09:25:08,345 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T09:25:08,346 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:08,347 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T09:25:08,347 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T09:25:08,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T09:25:08,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T09:25:08,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:08,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:08,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T09:25:08,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T09:25:08,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:08,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:08,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T09:25:08,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T09:25:08,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:08,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:08,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T09:25:08,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T09:25:08,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:08,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T09:25:08,372 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T09:25:08,375 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740 2024-11-10T09:25:08,378 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740 2024-11-10T09:25:08,381 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T09:25:08,382 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T09:25:08,383 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:25:08,386 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T09:25:08,389 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63863412, jitterRate=-0.048361003398895264}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:25:08,389 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T09:25:08,392 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731230708347Writing region info on filesystem at 1731230708347Initializing all the Stores at 1731230708350 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230708350Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230708352 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230708352Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731230708352Cleaning up temporary data from old regions at 1731230708382 (+30 ms)Running coprocessor post-open hooks at 1731230708390 (+8 ms)Region opened successfully at 1731230708392 (+2 ms) 2024-11-10T09:25:08,402 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731230708262 2024-11-10T09:25:08,419 DEBUG [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T09:25:08,420 INFO [RS_OPEN_META-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T09:25:08,421 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:08,424 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 802346a334f5,32883,1731230704436, state=OPEN 2024-11-10T09:25:08,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T09:25:08,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T09:25:08,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:25:08,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T09:25:08,527 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:08,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T09:25:08,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=802346a334f5,32883,1731230704436 in 432 msec 2024-11-10T09:25:08,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T09:25:08,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 693 msec 2024-11-10T09:25:08,552 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T09:25:08,552 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T09:25:08,577 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:08,579 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:08,601 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:08,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51803, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:08,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4200 sec 2024-11-10T09:25:08,635 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731230708635, completionTime=-1 2024-11-10T09:25:08,638 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T09:25:08,639 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T09:25:08,713 INFO [master/802346a334f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T09:25:08,713 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731230768713 2024-11-10T09:25:08,713 INFO [master/802346a334f5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731230828713 2024-11-10T09:25:08,713 INFO [master/802346a334f5:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 74 msec 2024-11-10T09:25:08,717 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38235,1731230703274-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:08,717 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38235,1731230703274-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:08,717 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38235,1731230703274-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:08,719 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-802346a334f5:38235, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:08,719 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:08,720 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:08,728 DEBUG [master/802346a334f5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T09:25:08,760 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.152sec 2024-11-10T09:25:08,761 INFO [master/802346a334f5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T09:25:08,763 INFO [master/802346a334f5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T09:25:08,764 INFO [master/802346a334f5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T09:25:08,765 INFO [master/802346a334f5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T09:25:08,765 INFO [master/802346a334f5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T09:25:08,766 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38235,1731230703274-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T09:25:08,767 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38235,1731230703274-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T09:25:08,775 DEBUG [master/802346a334f5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T09:25:08,776 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T09:25:08,776 INFO [master/802346a334f5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=802346a334f5,38235,1731230703274-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T09:25:08,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57a2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:08,861 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T09:25:08,862 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T09:25:08,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:08,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:08,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a49088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:08,888 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:08,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7fbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:08,892 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:08,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:08,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=802346a334f5,38235,1731230703274 2024-11-10T09:25:08,940 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=666, ProcessCount=11, AvailableMemoryMB=7793 2024-11-10T09:25:08,950 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:25:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,38235,1731230703274 2024-11-10T09:25:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@49a86ad1 2024-11-10T09:25:08,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:25:08,963 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:25:08,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:25:08,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:25:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:08,983 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:25:08,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T09:25:08,987 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:08,989 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:25:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:25:09,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741835_1011 (size=959) 2024-11-10T09:25:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:25:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:25:09,408 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:25:09,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741836_1012 (size=53) 2024-11-10T09:25:09,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:09,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 62a21c798fe3504a706a57ba0baaaff3, disabling compactions & flushes 2024-11-10T09:25:09,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:09,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:09,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. after waiting 0 ms 2024-11-10T09:25:09,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:09,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:09,423 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 62a21c798fe3504a706a57ba0baaaff3: Waiting for close lock at 1731230709422Disabling compacts and flushes for region at 1731230709422Disabling writes for close at 1731230709422Writing region close event to WAL at 1731230709422Closed at 1731230709422 2024-11-10T09:25:09,425 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:25:09,431 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230709426"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230709426"}]},"ts":"1731230709426"} 2024-11-10T09:25:09,436 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:25:09,440 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:25:09,442 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230709440"}]},"ts":"1731230709440"} 2024-11-10T09:25:09,447 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:25:09,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, ASSIGN}] 2024-11-10T09:25:09,451 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, ASSIGN 2024-11-10T09:25:09,452 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, ASSIGN; state=OFFLINE, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=false 2024-11-10T09:25:09,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=62a21c798fe3504a706a57ba0baaaff3, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:09,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, ASSIGN because future has completed 2024-11-10T09:25:09,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:25:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:25:09,773 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:09,773 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:25:09,774 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,774 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:09,774 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,774 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,777 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,781 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a21c798fe3504a706a57ba0baaaff3 columnFamilyName A 2024-11-10T09:25:09,781 DEBUG [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:09,784 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] regionserver.HStore(327): Store=62a21c798fe3504a706a57ba0baaaff3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:09,785 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,788 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a21c798fe3504a706a57ba0baaaff3 columnFamilyName B 2024-11-10T09:25:09,788 DEBUG [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:09,790 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] regionserver.HStore(327): Store=62a21c798fe3504a706a57ba0baaaff3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:09,790 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,793 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a21c798fe3504a706a57ba0baaaff3 columnFamilyName C 2024-11-10T09:25:09,793 DEBUG [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:09,794 INFO [StoreOpener-62a21c798fe3504a706a57ba0baaaff3-1 {}] regionserver.HStore(327): Store=62a21c798fe3504a706a57ba0baaaff3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:09,794 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,796 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,797 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,799 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,800 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,801 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:25:09,804 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,810 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:25:09,811 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 62a21c798fe3504a706a57ba0baaaff3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66037330, jitterRate=-0.015967100858688354}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:25:09,811 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:09,813 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 62a21c798fe3504a706a57ba0baaaff3: Running coprocessor pre-open hook at 1731230709774Writing region info on filesystem at 1731230709775 (+1 ms)Initializing all the Stores at 1731230709777 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230709777Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230709777Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230709777Cleaning up temporary data from old regions at 1731230709800 (+23 ms)Running coprocessor post-open hooks at 1731230709812 (+12 ms)Region opened successfully at 1731230709813 (+1 ms) 2024-11-10T09:25:09,815 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., pid=6, masterSystemTime=1731230709765 2024-11-10T09:25:09,820 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:09,820 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:09,822 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=62a21c798fe3504a706a57ba0baaaff3, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:09,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:25:09,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T09:25:09,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 in 219 msec 2024-11-10T09:25:09,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T09:25:09,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, ASSIGN in 387 msec 2024-11-10T09:25:09,842 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:25:09,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230709842"}]},"ts":"1731230709842"} 2024-11-10T09:25:09,846 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:25:09,848 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:25:09,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 876 msec 2024-11-10T09:25:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T09:25:10,138 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:10,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487a66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ede97e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,149 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc70ab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,152 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,160 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:25:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,38235,1731230703274 2024-11-10T09:25:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1820fead 2024-11-10T09:25:10,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:25:10,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:25:10,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,174 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7def95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,180 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@80438ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7907fcac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fc0926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,197 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac8d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,202 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797de42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a888f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,215 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c90bd87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813eeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,220 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c9f602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,224 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1853672d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562d1bb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,233 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae051c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225d1960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,236 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b367dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327794d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,243 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e9a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,245 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,246 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1058791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7449cb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:10,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:10,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b6d532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,250 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:10,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20994eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:10,256 DEBUG [Thread-161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,256 DEBUG [Thread-157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,256 DEBUG [Thread-150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,257 DEBUG [Thread-159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,261 DEBUG [Thread-144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,263 DEBUG [Thread-155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T09:25:10,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,265 DEBUG [Thread-146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,269 DEBUG [Thread-152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,269 DEBUG [Thread-148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:25:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,273 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:10,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:10,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,278 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:25:10,365 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:10,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:25:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:25:10,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:10,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:10,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:10,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e79e8d504998498ebb07b58320f558b1 is 50, key is test_row_0/A:col10/1731230710350/Put/seqid=0 2024-11-10T09:25:10,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741837_1013 (size=14341) 2024-11-10T09:25:10,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e79e8d504998498ebb07b58320f558b1 2024-11-10T09:25:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:25:10,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230770576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230770568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230770586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230770595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230770586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/ef496a1a80064361b0ceac504626b3a0 is 50, key is test_row_0/B:col10/1731230710350/Put/seqid=0 2024-11-10T09:25:10,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:10,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:10,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:10,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:10,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741838_1014 (size=12001) 2024-11-10T09:25:10,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/ef496a1a80064361b0ceac504626b3a0 2024-11-10T09:25:10,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230770737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230770737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230770738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230770739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230770747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:10,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:10,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9413ebcd82d146b4a673e1b5e1f66b6b is 50, key is test_row_0/C:col10/1731230710350/Put/seqid=0 2024-11-10T09:25:10,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741839_1015 (size=12001) 2024-11-10T09:25:10,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9413ebcd82d146b4a673e1b5e1f66b6b 2024-11-10T09:25:10,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e79e8d504998498ebb07b58320f558b1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e79e8d504998498ebb07b58320f558b1 2024-11-10T09:25:10,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e79e8d504998498ebb07b58320f558b1, entries=200, sequenceid=13, filesize=14.0 K 2024-11-10T09:25:10,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/ef496a1a80064361b0ceac504626b3a0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ef496a1a80064361b0ceac504626b3a0 2024-11-10T09:25:10,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:10,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:10,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:10,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:10,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:10,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ef496a1a80064361b0ceac504626b3a0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:25:10,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9413ebcd82d146b4a673e1b5e1f66b6b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9413ebcd82d146b4a673e1b5e1f66b6b 2024-11-10T09:25:10,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9413ebcd82d146b4a673e1b5e1f66b6b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:25:10,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 62a21c798fe3504a706a57ba0baaaff3 in 508ms, sequenceid=13, compaction requested=false 2024-11-10T09:25:10,881 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T09:25:10,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:25:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:10,962 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:10,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:25:10,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/8cbfaf1722e54e338443592a726293f8 is 50, key is test_row_0/A:col10/1731230710959/Put/seqid=0 2024-11-10T09:25:10,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741840_1016 (size=16681) 2024-11-10T09:25:11,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:11,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/8cbfaf1722e54e338443592a726293f8 2024-11-10T09:25:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:11,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/94dfa4e719144950afbb74ba0f5d5e12 is 50, key is test_row_0/B:col10/1731230710959/Put/seqid=0 2024-11-10T09:25:11,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741841_1017 (size=12001) 2024-11-10T09:25:11,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230771041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230771042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230771046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230771047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230771055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:11,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230771167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230771167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230771168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230771168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230771176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:11,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:11,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230771387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230771388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230771387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230771388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230771397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:25:11,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/94dfa4e719144950afbb74ba0f5d5e12 2024-11-10T09:25:11,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:11,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:11,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:11,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8cb9d60230c7488dacece929683ef7cd is 50, key is test_row_0/C:col10/1731230710959/Put/seqid=0 2024-11-10T09:25:11,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741842_1018 (size=12001) 2024-11-10T09:25:11,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8cb9d60230c7488dacece929683ef7cd 2024-11-10T09:25:11,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/8cbfaf1722e54e338443592a726293f8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/8cbfaf1722e54e338443592a726293f8 2024-11-10T09:25:11,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/8cbfaf1722e54e338443592a726293f8, entries=250, sequenceid=39, filesize=16.3 K 2024-11-10T09:25:11,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/94dfa4e719144950afbb74ba0f5d5e12 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/94dfa4e719144950afbb74ba0f5d5e12 2024-11-10T09:25:11,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/94dfa4e719144950afbb74ba0f5d5e12, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T09:25:11,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8cb9d60230c7488dacece929683ef7cd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8cb9d60230c7488dacece929683ef7cd 2024-11-10T09:25:11,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8cb9d60230c7488dacece929683ef7cd, entries=150, sequenceid=39, filesize=11.7 K 2024-11-10T09:25:11,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 62a21c798fe3504a706a57ba0baaaff3 in 616ms, sequenceid=39, compaction requested=false 2024-11-10T09:25:11,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:11,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T09:25:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,633 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T09:25:11,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/f4170317d521420a941addf6eb8f68ac is 50, key is test_row_0/A:col10/1731230710986/Put/seqid=0 2024-11-10T09:25:11,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741843_1019 (size=12001) 2024-11-10T09:25:11,680 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/f4170317d521420a941addf6eb8f68ac 2024-11-10T09:25:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:11,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/81ee1b964aa0435882abccc50322989c is 50, key is test_row_0/B:col10/1731230710986/Put/seqid=0 2024-11-10T09:25:11,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741844_1020 (size=12001) 2024-11-10T09:25:11,749 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/81ee1b964aa0435882abccc50322989c 2024-11-10T09:25:11,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230771763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230771767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/2a32f6b5fcbb4df99062c9e718314435 is 50, key is test_row_0/C:col10/1731230710986/Put/seqid=0 2024-11-10T09:25:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230771768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230771773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230771777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741845_1021 (size=12001) 2024-11-10T09:25:11,799 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/2a32f6b5fcbb4df99062c9e718314435 2024-11-10T09:25:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/f4170317d521420a941addf6eb8f68ac as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f4170317d521420a941addf6eb8f68ac 2024-11-10T09:25:11,832 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f4170317d521420a941addf6eb8f68ac, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:25:11,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/81ee1b964aa0435882abccc50322989c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81ee1b964aa0435882abccc50322989c 2024-11-10T09:25:11,854 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81ee1b964aa0435882abccc50322989c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:25:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/2a32f6b5fcbb4df99062c9e718314435 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2a32f6b5fcbb4df99062c9e718314435 2024-11-10T09:25:11,871 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2a32f6b5fcbb4df99062c9e718314435, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:25:11,873 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 62a21c798fe3504a706a57ba0baaaff3 in 240ms, sequenceid=49, compaction requested=true 2024-11-10T09:25:11,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:11,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:11,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T09:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T09:25:11,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T09:25:11,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5980 sec 2024-11-10T09:25:11,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6240 sec 2024-11-10T09:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:11,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:11,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T09:25:11,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b443cbf5c1974412b2ef8b33824d32c1 is 50, key is test_row_0/A:col10/1731230711766/Put/seqid=0 2024-11-10T09:25:11,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230771906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230771907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230771907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230771915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230771920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:11,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741846_1022 (size=12001) 2024-11-10T09:25:11,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b443cbf5c1974412b2ef8b33824d32c1 2024-11-10T09:25:11,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/70bcc9077d3e429da15fbc1b7d4e2de5 is 50, key is test_row_0/B:col10/1731230711766/Put/seqid=0 2024-11-10T09:25:12,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741847_1023 (size=12001) 2024-11-10T09:25:12,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/70bcc9077d3e429da15fbc1b7d4e2de5 2024-11-10T09:25:12,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230772027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230772027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230772027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230772036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230772038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/48e3d47380914521a759f362c3339fee is 50, key is test_row_0/C:col10/1731230711766/Put/seqid=0 2024-11-10T09:25:12,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741848_1024 (size=12001) 2024-11-10T09:25:12,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/48e3d47380914521a759f362c3339fee 2024-11-10T09:25:12,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b443cbf5c1974412b2ef8b33824d32c1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b443cbf5c1974412b2ef8b33824d32c1 2024-11-10T09:25:12,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b443cbf5c1974412b2ef8b33824d32c1, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T09:25:12,119 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T09:25:12,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/70bcc9077d3e429da15fbc1b7d4e2de5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70bcc9077d3e429da15fbc1b7d4e2de5 2024-11-10T09:25:12,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70bcc9077d3e429da15fbc1b7d4e2de5, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T09:25:12,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/48e3d47380914521a759f362c3339fee as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/48e3d47380914521a759f362c3339fee 2024-11-10T09:25:12,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/48e3d47380914521a759f362c3339fee, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T09:25:12,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 62a21c798fe3504a706a57ba0baaaff3 in 263ms, sequenceid=76, compaction requested=true 2024-11-10T09:25:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:12,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:12,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:12,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:12,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:12,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:12,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:12,175 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:12,178 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:12,181 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:12,183 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:12,183 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:12,183 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:12,184 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:12,184 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:12,184 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ef496a1a80064361b0ceac504626b3a0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/94dfa4e719144950afbb74ba0f5d5e12, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81ee1b964aa0435882abccc50322989c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70bcc9077d3e429da15fbc1b7d4e2de5] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=46.9 K 2024-11-10T09:25:12,184 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e79e8d504998498ebb07b58320f558b1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/8cbfaf1722e54e338443592a726293f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f4170317d521420a941addf6eb8f68ac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b443cbf5c1974412b2ef8b33824d32c1] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=53.7 K 2024-11-10T09:25:12,186 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ef496a1a80064361b0ceac504626b3a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230710323 2024-11-10T09:25:12,187 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e79e8d504998498ebb07b58320f558b1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230710323 2024-11-10T09:25:12,193 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 94dfa4e719144950afbb74ba0f5d5e12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731230710523 2024-11-10T09:25:12,194 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ee1b964aa0435882abccc50322989c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230710986 2024-11-10T09:25:12,195 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 70bcc9077d3e429da15fbc1b7d4e2de5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230711766 2024-11-10T09:25:12,196 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cbfaf1722e54e338443592a726293f8, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731230710523 2024-11-10T09:25:12,198 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4170317d521420a941addf6eb8f68ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230710986 2024-11-10T09:25:12,200 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting b443cbf5c1974412b2ef8b33824d32c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230711766 2024-11-10T09:25:12,251 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#12 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:12,252 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/d94ee52b31ae48d5846b2d7b87dd2cf3 is 50, key is test_row_0/A:col10/1731230711766/Put/seqid=0 2024-11-10T09:25:12,263 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#13 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:12,264 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c60bf496e6034be89cc7c3b11f25e83e is 50, key is test_row_0/B:col10/1731230711766/Put/seqid=0 2024-11-10T09:25:12,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741849_1025 (size=12139) 2024-11-10T09:25:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:12,274 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:12,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:25:12,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741850_1026 (size=12139) 2024-11-10T09:25:12,295 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/d94ee52b31ae48d5846b2d7b87dd2cf3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/d94ee52b31ae48d5846b2d7b87dd2cf3 2024-11-10T09:25:12,334 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into d94ee52b31ae48d5846b2d7b87dd2cf3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:12,334 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:12,337 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=12, startTime=1731230712162; duration=0sec 2024-11-10T09:25:12,337 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:12,338 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:12,338 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:12,343 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:12,343 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:12,343 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:12,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/c491ca41bf7b49779a41c93fe862d018 is 50, key is test_row_0/A:col10/1731230712258/Put/seqid=0 2024-11-10T09:25:12,344 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9413ebcd82d146b4a673e1b5e1f66b6b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8cb9d60230c7488dacece929683ef7cd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2a32f6b5fcbb4df99062c9e718314435, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/48e3d47380914521a759f362c3339fee] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=46.9 K 2024-11-10T09:25:12,347 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9413ebcd82d146b4a673e1b5e1f66b6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230710323 2024-11-10T09:25:12,350 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cb9d60230c7488dacece929683ef7cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731230710523 2024-11-10T09:25:12,352 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a32f6b5fcbb4df99062c9e718314435, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230710986 2024-11-10T09:25:12,353 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e3d47380914521a759f362c3339fee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230711766 2024-11-10T09:25:12,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741851_1027 (size=12001) 2024-11-10T09:25:12,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/c491ca41bf7b49779a41c93fe862d018 2024-11-10T09:25:12,396 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#15 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:12,398 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/06ae6e752034449bac3d24e82ed3a93f is 50, key is test_row_0/C:col10/1731230711766/Put/seqid=0 2024-11-10T09:25:12,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0bdd9fd551b841eab14f7390c1bc34c5 is 50, key is test_row_0/B:col10/1731230712258/Put/seqid=0 2024-11-10T09:25:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741852_1028 (size=12139) 2024-11-10T09:25:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T09:25:12,416 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:12,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T09:25:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:25:12,426 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:12,427 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:12,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:12,435 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/06ae6e752034449bac3d24e82ed3a93f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/06ae6e752034449bac3d24e82ed3a93f 2024-11-10T09:25:12,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230772433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,453 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 06ae6e752034449bac3d24e82ed3a93f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:12,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,453 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:12,453 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=12, startTime=1731230712171; duration=0sec 2024-11-10T09:25:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230772434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,453 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:12,453 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230772438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230772439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741853_1029 (size=12001) 2024-11-10T09:25:12,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0bdd9fd551b841eab14f7390c1bc34c5 2024-11-10T09:25:12,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230772452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/377796ce6ef84bebbf422cba12c28ddc is 50, key is test_row_0/C:col10/1731230712258/Put/seqid=0 2024-11-10T09:25:12,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741854_1030 (size=12001) 2024-11-10T09:25:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:25:12,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/377796ce6ef84bebbf422cba12c28ddc 2024-11-10T09:25:12,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/c491ca41bf7b49779a41c93fe862d018 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c491ca41bf7b49779a41c93fe862d018 2024-11-10T09:25:12,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230772556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230772556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c491ca41bf7b49779a41c93fe862d018, entries=150, sequenceid=87, filesize=11.7 K 2024-11-10T09:25:12,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0bdd9fd551b841eab14f7390c1bc34c5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bdd9fd551b841eab14f7390c1bc34c5 2024-11-10T09:25:12,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230772567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230772568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230772576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:25:12,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:12,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:12,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:12,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:12,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bdd9fd551b841eab14f7390c1bc34c5, entries=150, sequenceid=87, filesize=11.7 K 2024-11-10T09:25:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/377796ce6ef84bebbf422cba12c28ddc as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/377796ce6ef84bebbf422cba12c28ddc 2024-11-10T09:25:12,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/377796ce6ef84bebbf422cba12c28ddc, entries=150, sequenceid=87, filesize=11.7 K 2024-11-10T09:25:12,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 62a21c798fe3504a706a57ba0baaaff3 in 338ms, sequenceid=87, compaction requested=false 2024-11-10T09:25:12,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:12,713 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c60bf496e6034be89cc7c3b11f25e83e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c60bf496e6034be89cc7c3b11f25e83e 2024-11-10T09:25:12,729 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into c60bf496e6034be89cc7c3b11f25e83e(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:12,729 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:12,729 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=12, startTime=1731230712171; duration=0sec 2024-11-10T09:25:12,729 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:12,730 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:25:12,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T09:25:12,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:12,738 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:25:12,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/43db33b024c7493f8394f3c499797d1a is 50, key is test_row_0/A:col10/1731230712434/Put/seqid=0 2024-11-10T09:25:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:12,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:12,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741855_1031 (size=12001) 2024-11-10T09:25:12,774 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/43db33b024c7493f8394f3c499797d1a 2024-11-10T09:25:12,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230772787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230772789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230772793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230772796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230772796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/70dff667220f40019cb8da6f4c7ce63b is 50, key is test_row_0/B:col10/1731230712434/Put/seqid=0 2024-11-10T09:25:12,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741856_1032 (size=12001) 2024-11-10T09:25:12,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230772906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230772906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230772907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230772907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:12,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230772917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:25:13,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230773116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230773116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230773126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230773127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230773136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,236 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/70dff667220f40019cb8da6f4c7ce63b 2024-11-10T09:25:13,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/67732838438a4aaabc91e3bb5b10208c is 50, key is test_row_0/C:col10/1731230712434/Put/seqid=0 2024-11-10T09:25:13,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741857_1033 (size=12001) 2024-11-10T09:25:13,278 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/67732838438a4aaabc91e3bb5b10208c 2024-11-10T09:25:13,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/43db33b024c7493f8394f3c499797d1a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/43db33b024c7493f8394f3c499797d1a 2024-11-10T09:25:13,332 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/43db33b024c7493f8394f3c499797d1a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T09:25:13,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/70dff667220f40019cb8da6f4c7ce63b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70dff667220f40019cb8da6f4c7ce63b 2024-11-10T09:25:13,352 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70dff667220f40019cb8da6f4c7ce63b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T09:25:13,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/67732838438a4aaabc91e3bb5b10208c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/67732838438a4aaabc91e3bb5b10208c 2024-11-10T09:25:13,379 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/67732838438a4aaabc91e3bb5b10208c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T09:25:13,380 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 62a21c798fe3504a706a57ba0baaaff3 in 642ms, sequenceid=115, compaction requested=true 2024-11-10T09:25:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:13,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T09:25:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T09:25:13,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T09:25:13,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 956 msec 2024-11-10T09:25:13,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 975 msec 2024-11-10T09:25:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:13,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:25:13,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/86388ce2631a41ad9f1c9afdb2866a5e is 50, key is test_row_0/A:col10/1731230712792/Put/seqid=0 2024-11-10T09:25:13,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741858_1034 (size=14341) 2024-11-10T09:25:13,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/86388ce2631a41ad9f1c9afdb2866a5e 2024-11-10T09:25:13,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/d64dd032cd0c40599619ec4cf1838c24 is 50, key is test_row_0/B:col10/1731230712792/Put/seqid=0 2024-11-10T09:25:13,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741859_1035 (size=12001) 2024-11-10T09:25:13,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/d64dd032cd0c40599619ec4cf1838c24 2024-11-10T09:25:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T09:25:13,556 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:13,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T09:25:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:25:13,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:13,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0db94f1fb7d34a17b78ecf040283aa84 is 50, key is test_row_0/C:col10/1731230712792/Put/seqid=0 2024-11-10T09:25:13,564 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:13,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:13,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741860_1036 (size=12001) 2024-11-10T09:25:13,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230773598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230773601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230773601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230773610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230773613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:25:13,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230773715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:25:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:13,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:13,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:13,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230773726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230773726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230773726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230773728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,869 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:25:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:25:13,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:25:13,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:13,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:13,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:13,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:13,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:13,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230773925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230773936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,939 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T09:25:13,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230773936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230773936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,942 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T09:25:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,942 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T09:25:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,945 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T09:25:13,945 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T09:25:13,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T09:25:13,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T09:25:13,948 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:25:13,948 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T09:25:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230773945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:13,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0db94f1fb7d34a17b78ecf040283aa84 2024-11-10T09:25:13,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/86388ce2631a41ad9f1c9afdb2866a5e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/86388ce2631a41ad9f1c9afdb2866a5e 2024-11-10T09:25:14,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/86388ce2631a41ad9f1c9afdb2866a5e, entries=200, sequenceid=128, filesize=14.0 K 2024-11-10T09:25:14,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/d64dd032cd0c40599619ec4cf1838c24 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/d64dd032cd0c40599619ec4cf1838c24 2024-11-10T09:25:14,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/d64dd032cd0c40599619ec4cf1838c24, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T09:25:14,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0db94f1fb7d34a17b78ecf040283aa84 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0db94f1fb7d34a17b78ecf040283aa84 2024-11-10T09:25:14,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:25:14,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:14,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:14,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:14,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0db94f1fb7d34a17b78ecf040283aa84, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T09:25:14,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 62a21c798fe3504a706a57ba0baaaff3 in 620ms, sequenceid=128, compaction requested=true 2024-11-10T09:25:14,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:14,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:14,054 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:14,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:14,054 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:14,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:14,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:14,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:14,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:14,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:14,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:14,061 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,061 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c60bf496e6034be89cc7c3b11f25e83e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bdd9fd551b841eab14f7390c1bc34c5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70dff667220f40019cb8da6f4c7ce63b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/d64dd032cd0c40599619ec4cf1838c24] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=47.0 K 2024-11-10T09:25:14,061 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50482 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:14,062 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:14,062 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,062 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/d94ee52b31ae48d5846b2d7b87dd2cf3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c491ca41bf7b49779a41c93fe862d018, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/43db33b024c7493f8394f3c499797d1a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/86388ce2631a41ad9f1c9afdb2866a5e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=49.3 K 2024-11-10T09:25:14,063 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c60bf496e6034be89cc7c3b11f25e83e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230711766 2024-11-10T09:25:14,066 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d94ee52b31ae48d5846b2d7b87dd2cf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230711766 2024-11-10T09:25:14,067 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bdd9fd551b841eab14f7390c1bc34c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731230711903 2024-11-10T09:25:14,067 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c491ca41bf7b49779a41c93fe862d018, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731230711903 2024-11-10T09:25:14,068 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 70dff667220f40019cb8da6f4c7ce63b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230712398 2024-11-10T09:25:14,069 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43db33b024c7493f8394f3c499797d1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230712398 2024-11-10T09:25:14,073 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d64dd032cd0c40599619ec4cf1838c24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731230712792 2024-11-10T09:25:14,075 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86388ce2631a41ad9f1c9afdb2866a5e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731230712792 2024-11-10T09:25:14,147 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:14,148 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4ce583dc9c114855a710b2594bb699f3 is 50, key is test_row_0/A:col10/1731230712792/Put/seqid=0 2024-11-10T09:25:14,150 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:14,151 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/608f811fd361465eba1f6302cc9b486c is 50, key is test_row_0/B:col10/1731230712792/Put/seqid=0 2024-11-10T09:25:14,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741861_1037 (size=12275) 2024-11-10T09:25:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:25:14,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T09:25:14,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,190 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:25:14,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741862_1038 (size=12275) 2024-11-10T09:25:14,209 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/608f811fd361465eba1f6302cc9b486c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/608f811fd361465eba1f6302cc9b486c 2024-11-10T09:25:14,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4a1a618f22bc4db88b3b224950ba7545 is 50, key is test_row_0/A:col10/1731230713597/Put/seqid=0 2024-11-10T09:25:14,225 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into 608f811fd361465eba1f6302cc9b486c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:14,226 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:14,226 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=12, startTime=1731230714054; duration=0sec 2024-11-10T09:25:14,226 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:14,226 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:14,226 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:14,228 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:14,228 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:14,228 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,228 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/06ae6e752034449bac3d24e82ed3a93f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/377796ce6ef84bebbf422cba12c28ddc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/67732838438a4aaabc91e3bb5b10208c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0db94f1fb7d34a17b78ecf040283aa84] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=47.0 K 2024-11-10T09:25:14,230 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ae6e752034449bac3d24e82ed3a93f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230711766 2024-11-10T09:25:14,231 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 377796ce6ef84bebbf422cba12c28ddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731230711903 2024-11-10T09:25:14,234 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 67732838438a4aaabc91e3bb5b10208c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230712398 2024-11-10T09:25:14,235 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db94f1fb7d34a17b78ecf040283aa84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731230712792 2024-11-10T09:25:14,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:14,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741863_1039 (size=12151) 2024-11-10T09:25:14,268 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4a1a618f22bc4db88b3b224950ba7545 2024-11-10T09:25:14,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230774264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230774268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230774268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230774271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,282 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:14,283 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/5afcea01cc374dee91b011184bb1d06b is 50, key is test_row_0/C:col10/1731230712792/Put/seqid=0 2024-11-10T09:25:14,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230774281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/810ac48bf3cc41d88a307e6ea58cda1b is 50, key is test_row_0/B:col10/1731230713597/Put/seqid=0 2024-11-10T09:25:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741864_1040 (size=12275) 2024-11-10T09:25:14,329 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/5afcea01cc374dee91b011184bb1d06b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5afcea01cc374dee91b011184bb1d06b 2024-11-10T09:25:14,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741865_1041 (size=12151) 2024-11-10T09:25:14,338 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/810ac48bf3cc41d88a307e6ea58cda1b 2024-11-10T09:25:14,351 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 5afcea01cc374dee91b011184bb1d06b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:14,351 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:14,351 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=12, startTime=1731230714054; duration=0sec 2024-11-10T09:25:14,351 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:14,352 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0c1f18be4f11461f9a1d229076cbce5c is 50, key is test_row_0/C:col10/1731230713597/Put/seqid=0 2024-11-10T09:25:14,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741866_1042 (size=12151) 2024-11-10T09:25:14,387 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0c1f18be4f11461f9a1d229076cbce5c 2024-11-10T09:25:14,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230774387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230774387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230774387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230774387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230774395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4a1a618f22bc4db88b3b224950ba7545 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4a1a618f22bc4db88b3b224950ba7545 2024-11-10T09:25:14,415 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4a1a618f22bc4db88b3b224950ba7545, entries=150, sequenceid=151, filesize=11.9 K 2024-11-10T09:25:14,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/810ac48bf3cc41d88a307e6ea58cda1b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/810ac48bf3cc41d88a307e6ea58cda1b 2024-11-10T09:25:14,432 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/810ac48bf3cc41d88a307e6ea58cda1b, entries=150, sequenceid=151, filesize=11.9 K 2024-11-10T09:25:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0c1f18be4f11461f9a1d229076cbce5c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c1f18be4f11461f9a1d229076cbce5c 2024-11-10T09:25:14,447 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c1f18be4f11461f9a1d229076cbce5c, entries=150, sequenceid=151, filesize=11.9 K 2024-11-10T09:25:14,451 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 62a21c798fe3504a706a57ba0baaaff3 in 260ms, sequenceid=151, compaction requested=false 2024-11-10T09:25:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T09:25:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T09:25:14,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T09:25:14,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 889 msec 2024-11-10T09:25:14,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 902 msec 2024-11-10T09:25:14,586 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4ce583dc9c114855a710b2594bb699f3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4ce583dc9c114855a710b2594bb699f3 2024-11-10T09:25:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:14,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:14,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T09:25:14,603 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into 4ce583dc9c114855a710b2594bb699f3(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:14,603 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:14,603 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=12, startTime=1731230714053; duration=0sec 2024-11-10T09:25:14,603 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:14,603 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:14,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e03e713283b548f09e78e96f6b08b670 is 50, key is test_row_0/A:col10/1731230714597/Put/seqid=0 2024-11-10T09:25:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741867_1043 (size=14541) 2024-11-10T09:25:14,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e03e713283b548f09e78e96f6b08b670 2024-11-10T09:25:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230774638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230774637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230774652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230774652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230774653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/598705ac1c5849ca9311fed6138a5202 is 50, key is test_row_0/B:col10/1731230714597/Put/seqid=0 2024-11-10T09:25:14,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741868_1044 (size=12151) 2024-11-10T09:25:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T09:25:14,696 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:14,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T09:25:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:25:14,701 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:14,703 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:14,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:14,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230774756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230774756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230774766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230774766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230774766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:25:14,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:25:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:14,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:14,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:14,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230774966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230774966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230774976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230774977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230774977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:14,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:25:15,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:25:15,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:15,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/598705ac1c5849ca9311fed6138a5202 2024-11-10T09:25:15,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/665641f3b40242c38232b9da95012fc8 is 50, key is test_row_0/C:col10/1731230714597/Put/seqid=0 2024-11-10T09:25:15,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741869_1045 (size=12151) 2024-11-10T09:25:15,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:25:15,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230775285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230775286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230775286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230775286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230775286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:25:15,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:25:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:15,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:25:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:15,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/665641f3b40242c38232b9da95012fc8 2024-11-10T09:25:15,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e03e713283b548f09e78e96f6b08b670 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e03e713283b548f09e78e96f6b08b670 2024-11-10T09:25:15,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e03e713283b548f09e78e96f6b08b670, entries=200, sequenceid=170, filesize=14.2 K 2024-11-10T09:25:15,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/598705ac1c5849ca9311fed6138a5202 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/598705ac1c5849ca9311fed6138a5202 2024-11-10T09:25:15,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/598705ac1c5849ca9311fed6138a5202, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T09:25:15,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/665641f3b40242c38232b9da95012fc8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/665641f3b40242c38232b9da95012fc8 2024-11-10T09:25:15,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/665641f3b40242c38232b9da95012fc8, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T09:25:15,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=127.47 KB/130530 for 62a21c798fe3504a706a57ba0baaaff3 in 974ms, sequenceid=170, compaction requested=true 2024-11-10T09:25:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:15,575 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:15,575 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:15,577 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:15,577 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:15,577 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:15,577 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:15,577 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,577 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,577 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4ce583dc9c114855a710b2594bb699f3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4a1a618f22bc4db88b3b224950ba7545, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e03e713283b548f09e78e96f6b08b670] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=38.1 K 2024-11-10T09:25:15,577 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/608f811fd361465eba1f6302cc9b486c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/810ac48bf3cc41d88a307e6ea58cda1b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/598705ac1c5849ca9311fed6138a5202] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=35.7 K 2024-11-10T09:25:15,578 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 608f811fd361465eba1f6302cc9b486c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731230712792 2024-11-10T09:25:15,578 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce583dc9c114855a710b2594bb699f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731230712792 2024-11-10T09:25:15,579 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a1a618f22bc4db88b3b224950ba7545, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731230713597 2024-11-10T09:25:15,580 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 810ac48bf3cc41d88a307e6ea58cda1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731230713597 2024-11-10T09:25:15,580 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e03e713283b548f09e78e96f6b08b670, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230714268 2024-11-10T09:25:15,582 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 598705ac1c5849ca9311fed6138a5202, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230714268 2024-11-10T09:25:15,609 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#33 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:15,610 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/f87f31a4b9944390a4dce26e0a194d95 is 50, key is test_row_0/A:col10/1731230714597/Put/seqid=0 2024-11-10T09:25:15,620 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:15,621 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/2faf6d57c7bd4efd841e63c60700de5d is 50, key is test_row_0/B:col10/1731230714597/Put/seqid=0 2024-11-10T09:25:15,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741870_1046 (size=12527) 2024-11-10T09:25:15,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T09:25:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,645 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:25:15,656 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/f87f31a4b9944390a4dce26e0a194d95 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f87f31a4b9944390a4dce26e0a194d95 2024-11-10T09:25:15,667 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into f87f31a4b9944390a4dce26e0a194d95(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:15,667 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:15,667 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=13, startTime=1731230715575; duration=0sec 2024-11-10T09:25:15,668 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:15,668 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:15,668 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:15,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741871_1047 (size=12527) 2024-11-10T09:25:15,671 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:15,671 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:15,671 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,671 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5afcea01cc374dee91b011184bb1d06b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c1f18be4f11461f9a1d229076cbce5c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/665641f3b40242c38232b9da95012fc8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=35.7 K 2024-11-10T09:25:15,673 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5afcea01cc374dee91b011184bb1d06b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731230712792 2024-11-10T09:25:15,674 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c1f18be4f11461f9a1d229076cbce5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731230713597 2024-11-10T09:25:15,675 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665641f3b40242c38232b9da95012fc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230714268 2024-11-10T09:25:15,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e92b76ec704b41268e29b4b396b1dd86 is 50, key is test_row_0/A:col10/1731230714647/Put/seqid=0 2024-11-10T09:25:15,680 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/2faf6d57c7bd4efd841e63c60700de5d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/2faf6d57c7bd4efd841e63c60700de5d 2024-11-10T09:25:15,693 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into 2faf6d57c7bd4efd841e63c60700de5d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:15,695 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:15,696 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=13, startTime=1731230715575; duration=0sec 2024-11-10T09:25:15,701 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:15,701 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:15,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741872_1048 (size=12151) 2024-11-10T09:25:15,706 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:15,707 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/40efca6cf57e4e8b915f21637507125f is 50, key is test_row_0/C:col10/1731230714597/Put/seqid=0 2024-11-10T09:25:15,709 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e92b76ec704b41268e29b4b396b1dd86 2024-11-10T09:25:15,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/192ed45510c249b9a1e30e6749ef1172 is 50, key is test_row_0/B:col10/1731230714647/Put/seqid=0 2024-11-10T09:25:15,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741873_1049 (size=12527) 2024-11-10T09:25:15,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741874_1050 (size=12151) 2024-11-10T09:25:15,748 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/192ed45510c249b9a1e30e6749ef1172 2024-11-10T09:25:15,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9c2065d2f9e34c5880a3167fe955cec3 is 50, key is test_row_0/C:col10/1731230714647/Put/seqid=0 2024-11-10T09:25:15,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741875_1051 (size=12151) 2024-11-10T09:25:15,791 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9c2065d2f9e34c5880a3167fe955cec3 2024-11-10T09:25:15,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e92b76ec704b41268e29b4b396b1dd86 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e92b76ec704b41268e29b4b396b1dd86 2024-11-10T09:25:15,816 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e92b76ec704b41268e29b4b396b1dd86, entries=150, sequenceid=192, filesize=11.9 K 2024-11-10T09:25:15,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/192ed45510c249b9a1e30e6749ef1172 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/192ed45510c249b9a1e30e6749ef1172 2024-11-10T09:25:15,830 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/192ed45510c249b9a1e30e6749ef1172, entries=150, sequenceid=192, filesize=11.9 K 2024-11-10T09:25:15,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9c2065d2f9e34c5880a3167fe955cec3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9c2065d2f9e34c5880a3167fe955cec3 2024-11-10T09:25:15,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230775821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:25:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230775829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230775831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230775833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,842 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9c2065d2f9e34c5880a3167fe955cec3, entries=150, sequenceid=192, filesize=11.9 K 2024-11-10T09:25:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,843 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 62a21c798fe3504a706a57ba0baaaff3 in 198ms, sequenceid=192, compaction requested=false 2024-11-10T09:25:15,844 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:15,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T09:25:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:15,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:15,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:15,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T09:25:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T09:25:15,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/aad0c17c62ea4c5ca302c6dbfc813a2d is 50, key is test_row_0/A:col10/1731230715825/Put/seqid=0 2024-11-10T09:25:15,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T09:25:15,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1470 sec 2024-11-10T09:25:15,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1670 sec 2024-11-10T09:25:15,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741876_1052 (size=14541) 2024-11-10T09:25:15,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/aad0c17c62ea4c5ca302c6dbfc813a2d 2024-11-10T09:25:15,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/e9817331c00b41f3b295db72dfc7db5b is 50, key is test_row_0/B:col10/1731230715825/Put/seqid=0 2024-11-10T09:25:15,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741877_1053 (size=12151) 2024-11-10T09:25:15,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/e9817331c00b41f3b295db72dfc7db5b 2024-11-10T09:25:15,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/f2ca59cf07684fea8422d31b11d8b2a8 is 50, key is test_row_0/C:col10/1731230715825/Put/seqid=0 2024-11-10T09:25:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230775946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230775949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230775951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230775954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230775954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:15,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741878_1054 (size=12151) 2024-11-10T09:25:15,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/f2ca59cf07684fea8422d31b11d8b2a8 2024-11-10T09:25:15,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/aad0c17c62ea4c5ca302c6dbfc813a2d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/aad0c17c62ea4c5ca302c6dbfc813a2d 2024-11-10T09:25:15,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/aad0c17c62ea4c5ca302c6dbfc813a2d, entries=200, sequenceid=210, filesize=14.2 K 2024-11-10T09:25:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/e9817331c00b41f3b295db72dfc7db5b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e9817331c00b41f3b295db72dfc7db5b 2024-11-10T09:25:16,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e9817331c00b41f3b295db72dfc7db5b, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T09:25:16,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/f2ca59cf07684fea8422d31b11d8b2a8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/f2ca59cf07684fea8422d31b11d8b2a8 2024-11-10T09:25:16,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/f2ca59cf07684fea8422d31b11d8b2a8, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T09:25:16,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 62a21c798fe3504a706a57ba0baaaff3 in 183ms, sequenceid=210, compaction requested=true 2024-11-10T09:25:16,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:16,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,027 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:16,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:16,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:16,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:16,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:25:16,030 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:16,031 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:16,031 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:16,031 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f87f31a4b9944390a4dce26e0a194d95, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e92b76ec704b41268e29b4b396b1dd86, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/aad0c17c62ea4c5ca302c6dbfc813a2d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=38.3 K 2024-11-10T09:25:16,033 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f87f31a4b9944390a4dce26e0a194d95, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230714268 2024-11-10T09:25:16,034 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e92b76ec704b41268e29b4b396b1dd86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731230714634 2024-11-10T09:25:16,036 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting aad0c17c62ea4c5ca302c6dbfc813a2d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230715825 2024-11-10T09:25:16,059 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:16,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/bd916235d93e4a75bcf8fbb3aa287a59 is 50, key is test_row_0/A:col10/1731230715825/Put/seqid=0 2024-11-10T09:25:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:16,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:25:16,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741879_1055 (size=12629) 2024-11-10T09:25:16,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/c38004d50f124174a168387b8bd3a8e6 is 50, key is test_row_0/A:col10/1731230716068/Put/seqid=0 2024-11-10T09:25:16,096 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/bd916235d93e4a75bcf8fbb3aa287a59 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bd916235d93e4a75bcf8fbb3aa287a59 2024-11-10T09:25:16,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741880_1056 (size=12151) 2024-11-10T09:25:16,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/c38004d50f124174a168387b8bd3a8e6 2024-11-10T09:25:16,109 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into bd916235d93e4a75bcf8fbb3aa287a59(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:16,109 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,109 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=13, startTime=1731230716027; duration=0sec 2024-11-10T09:25:16,109 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:25:16,109 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:16,109 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-10T09:25:16,113 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:25:16,114 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:25:16,115 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. because compaction request was cancelled 2024-11-10T09:25:16,115 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:16,115 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230776103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230776103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,118 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:16,118 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:16,118 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:16,119 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/2faf6d57c7bd4efd841e63c60700de5d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/192ed45510c249b9a1e30e6749ef1172, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e9817331c00b41f3b295db72dfc7db5b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=36.0 K 2024-11-10T09:25:16,119 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 2faf6d57c7bd4efd841e63c60700de5d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230714268 2024-11-10T09:25:16,120 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 192ed45510c249b9a1e30e6749ef1172, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731230714634 2024-11-10T09:25:16,121 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e9817331c00b41f3b295db72dfc7db5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230715825 2024-11-10T09:25:16,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/585733090e174c2bb8d659097ff48ff3 is 50, key is test_row_0/B:col10/1731230716068/Put/seqid=0 2024-11-10T09:25:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230776111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230776111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,147 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:16,148 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/7870eea5e20c416586323074de8de536 is 50, key is test_row_0/B:col10/1731230715825/Put/seqid=0 2024-11-10T09:25:16,153 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/40efca6cf57e4e8b915f21637507125f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/40efca6cf57e4e8b915f21637507125f 2024-11-10T09:25:16,165 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 40efca6cf57e4e8b915f21637507125f(size=12.2 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:16,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741881_1057 (size=12151) 2024-11-10T09:25:16,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/585733090e174c2bb8d659097ff48ff3 2024-11-10T09:25:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230776166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741882_1058 (size=12629) 2024-11-10T09:25:16,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/babb3899565e45f9a772f24093f52563 is 50, key is test_row_0/C:col10/1731230716068/Put/seqid=0 2024-11-10T09:25:16,195 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/7870eea5e20c416586323074de8de536 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/7870eea5e20c416586323074de8de536 2024-11-10T09:25:16,165 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,196 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=13, startTime=1731230715575; duration=0sec 2024-11-10T09:25:16,197 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,197 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:16,216 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into 7870eea5e20c416586323074de8de536(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:16,216 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,217 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=13, startTime=1731230716027; duration=0sec 2024-11-10T09:25:16,217 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,217 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230776226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230776226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741883_1059 (size=12151) 2024-11-10T09:25:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/babb3899565e45f9a772f24093f52563 2024-11-10T09:25:16,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230776238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230776238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/c38004d50f124174a168387b8bd3a8e6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c38004d50f124174a168387b8bd3a8e6 2024-11-10T09:25:16,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c38004d50f124174a168387b8bd3a8e6, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T09:25:16,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/585733090e174c2bb8d659097ff48ff3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/585733090e174c2bb8d659097ff48ff3 2024-11-10T09:25:16,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/585733090e174c2bb8d659097ff48ff3, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T09:25:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/babb3899565e45f9a772f24093f52563 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/babb3899565e45f9a772f24093f52563 2024-11-10T09:25:16,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/babb3899565e45f9a772f24093f52563, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T09:25:16,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 62a21c798fe3504a706a57ba0baaaff3 in 221ms, sequenceid=233, compaction requested=true 2024-11-10T09:25:16,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,291 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:25:16,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:16,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:16,295 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:25:16,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:16,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:16,296 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:25:16,296 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:25:16,296 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. because compaction request was cancelled 2024-11-10T09:25:16,296 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:16,296 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:16,296 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:25:16,296 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:25:16,297 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. because compaction request was cancelled 2024-11-10T09:25:16,297 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:16,299 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:16,299 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:16,299 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:16,299 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/40efca6cf57e4e8b915f21637507125f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9c2065d2f9e34c5880a3167fe955cec3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/f2ca59cf07684fea8422d31b11d8b2a8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/babb3899565e45f9a772f24093f52563] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=47.8 K 2024-11-10T09:25:16,300 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40efca6cf57e4e8b915f21637507125f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230714268 2024-11-10T09:25:16,302 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2065d2f9e34c5880a3167fe955cec3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731230714634 2024-11-10T09:25:16,303 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ca59cf07684fea8422d31b11d8b2a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230715825 2024-11-10T09:25:16,304 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting babb3899565e45f9a772f24093f52563, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731230715940 2024-11-10T09:25:16,332 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#47 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:16,334 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/606c2b87e4074d5596af728df49c5340 is 50, key is test_row_0/C:col10/1731230716068/Put/seqid=0 2024-11-10T09:25:16,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741884_1060 (size=12663) 2024-11-10T09:25:16,364 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/606c2b87e4074d5596af728df49c5340 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/606c2b87e4074d5596af728df49c5340 2024-11-10T09:25:16,375 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 606c2b87e4074d5596af728df49c5340(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:16,376 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,376 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=12, startTime=1731230716296; duration=0sec 2024-11-10T09:25:16,376 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,376 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:16,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:16,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T09:25:16,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/bc5e894fc4c64c16a8155c69520a4c31 is 50, key is test_row_0/A:col10/1731230716092/Put/seqid=0 2024-11-10T09:25:16,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741885_1061 (size=14541) 2024-11-10T09:25:16,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/bc5e894fc4c64c16a8155c69520a4c31 2024-11-10T09:25:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230776480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230776485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230776486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230776491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230776491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3d60c06f1e884eef8364c58b56c0cfb5 is 50, key is test_row_0/B:col10/1731230716092/Put/seqid=0 2024-11-10T09:25:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741886_1062 (size=12151) 2024-11-10T09:25:16,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3d60c06f1e884eef8364c58b56c0cfb5 2024-11-10T09:25:16,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c87877506192451aa679a12491f257af is 50, key is test_row_0/C:col10/1731230716092/Put/seqid=0 2024-11-10T09:25:16,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741887_1063 (size=12151) 2024-11-10T09:25:16,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c87877506192451aa679a12491f257af 2024-11-10T09:25:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/bc5e894fc4c64c16a8155c69520a4c31 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bc5e894fc4c64c16a8155c69520a4c31 2024-11-10T09:25:16,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bc5e894fc4c64c16a8155c69520a4c31, entries=200, sequenceid=252, filesize=14.2 K 2024-11-10T09:25:16,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3d60c06f1e884eef8364c58b56c0cfb5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3d60c06f1e884eef8364c58b56c0cfb5 2024-11-10T09:25:16,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3d60c06f1e884eef8364c58b56c0cfb5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T09:25:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c87877506192451aa679a12491f257af as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c87877506192451aa679a12491f257af 2024-11-10T09:25:16,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230776596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230776596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c87877506192451aa679a12491f257af, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T09:25:16,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 62a21c798fe3504a706a57ba0baaaff3 in 165ms, sequenceid=252, compaction requested=true 2024-11-10T09:25:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:16,605 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:16,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,605 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:16,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:16,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:16,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:16,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:16,607 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:16,607 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:16,607 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:16,607 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:16,607 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:16,607 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:16,607 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bd916235d93e4a75bcf8fbb3aa287a59, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c38004d50f124174a168387b8bd3a8e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bc5e894fc4c64c16a8155c69520a4c31] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=38.4 K 2024-11-10T09:25:16,607 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/7870eea5e20c416586323074de8de536, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/585733090e174c2bb8d659097ff48ff3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3d60c06f1e884eef8364c58b56c0cfb5] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=36.1 K 2024-11-10T09:25:16,608 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd916235d93e4a75bcf8fbb3aa287a59, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230715825 2024-11-10T09:25:16,608 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7870eea5e20c416586323074de8de536, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230715825 2024-11-10T09:25:16,608 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 585733090e174c2bb8d659097ff48ff3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731230715940 2024-11-10T09:25:16,609 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38004d50f124174a168387b8bd3a8e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731230715940 2024-11-10T09:25:16,609 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d60c06f1e884eef8364c58b56c0cfb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230716092 2024-11-10T09:25:16,611 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc5e894fc4c64c16a8155c69520a4c31, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230716092 2024-11-10T09:25:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:16,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:16,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:25:16,627 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:16,628 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c13987ce2fbf4270be331c6d4071abed is 50, key is test_row_0/B:col10/1731230716092/Put/seqid=0 2024-11-10T09:25:16,629 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:16,629 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/75ac015e77e14875add11fd665366812 is 50, key is test_row_0/A:col10/1731230716092/Put/seqid=0 2024-11-10T09:25:16,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4b2eecad80724be693626a133030b7ac is 50, key is test_row_0/A:col10/1731230716609/Put/seqid=0 2024-11-10T09:25:16,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741889_1065 (size=12731) 2024-11-10T09:25:16,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230776665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,677 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/75ac015e77e14875add11fd665366812 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/75ac015e77e14875add11fd665366812 2024-11-10T09:25:16,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230776678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,686 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into 75ac015e77e14875add11fd665366812(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:16,686 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:16,687 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=13, startTime=1731230716605; duration=0sec 2024-11-10T09:25:16,687 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:16,687 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:16,687 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:25:16,688 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:25:16,688 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:25:16,688 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. because compaction request was cancelled 2024-11-10T09:25:16,688 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741888_1064 (size=12731) 2024-11-10T09:25:16,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741890_1066 (size=14741) 2024-11-10T09:25:16,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230776776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230776796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230776806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230776806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T09:25:16,856 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:16,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T09:25:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:25:16,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:16,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:16,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:25:16,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230776986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:16,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230777005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230777015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4b2eecad80724be693626a133030b7ac 2024-11-10T09:25:17,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c13987ce2fbf4270be331c6d4071abed as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c13987ce2fbf4270be331c6d4071abed 2024-11-10T09:25:17,118 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into c13987ce2fbf4270be331c6d4071abed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:17,119 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:17,119 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=13, startTime=1731230716605; duration=0sec 2024-11-10T09:25:17,119 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:17,119 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:17,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3434beed70a04d0dbea25e28f39b7aea is 50, key is test_row_0/B:col10/1731230716609/Put/seqid=0 2024-11-10T09:25:17,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230777116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230777116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741891_1067 (size=12301) 2024-11-10T09:25:17,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3434beed70a04d0dbea25e28f39b7aea 2024-11-10T09:25:17,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/d9045ea43c2a4aeba788db5ce87d180a is 50, key is test_row_0/C:col10/1731230716609/Put/seqid=0 2024-11-10T09:25:17,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741892_1068 (size=12301) 2024-11-10T09:25:17,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:17,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:17,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:25:17,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230777295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:17,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:17,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230777326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:17,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:17,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:25:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/d9045ea43c2a4aeba788db5ce87d180a 2024-11-10T09:25:17,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/4b2eecad80724be693626a133030b7ac as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4b2eecad80724be693626a133030b7ac 2024-11-10T09:25:17,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4b2eecad80724be693626a133030b7ac, entries=200, sequenceid=277, filesize=14.4 K 2024-11-10T09:25:17,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3434beed70a04d0dbea25e28f39b7aea as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3434beed70a04d0dbea25e28f39b7aea 2024-11-10T09:25:17,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3434beed70a04d0dbea25e28f39b7aea, entries=150, sequenceid=277, filesize=12.0 K 2024-11-10T09:25:17,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/d9045ea43c2a4aeba788db5ce87d180a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/d9045ea43c2a4aeba788db5ce87d180a 2024-11-10T09:25:17,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/d9045ea43c2a4aeba788db5ce87d180a, entries=150, sequenceid=277, filesize=12.0 K 2024-11-10T09:25:17,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 62a21c798fe3504a706a57ba0baaaff3 in 979ms, sequenceid=277, compaction requested=true 2024-11-10T09:25:17,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:17,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:17,600 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:25:17,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:17,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:17,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:17,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:17,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:25:17,601 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:17,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:25:17,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:25:17,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. because compaction request was cancelled 2024-11-10T09:25:17,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:17,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:25:17,603 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:17,603 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:17,603 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,603 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/606c2b87e4074d5596af728df49c5340, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c87877506192451aa679a12491f257af, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/d9045ea43c2a4aeba788db5ce87d180a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=36.2 K 2024-11-10T09:25:17,604 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:25:17,604 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:25:17,604 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. because compaction request was cancelled 2024-11-10T09:25:17,604 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:17,604 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 606c2b87e4074d5596af728df49c5340, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731230715940 2024-11-10T09:25:17,605 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c87877506192451aa679a12491f257af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230716092 2024-11-10T09:25:17,606 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d9045ea43c2a4aeba788db5ce87d180a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230716609 2024-11-10T09:25:17,616 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:17,617 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/448e135a9633443282fd1b2bb49fc40c is 50, key is test_row_0/C:col10/1731230716609/Put/seqid=0 2024-11-10T09:25:17,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741893_1069 (size=12915) 2024-11-10T09:25:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:17,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:17,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:25:17,633 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/448e135a9633443282fd1b2bb49fc40c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/448e135a9633443282fd1b2bb49fc40c 2024-11-10T09:25:17,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:17,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:17,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,650 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 448e135a9633443282fd1b2bb49fc40c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:17,650 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:17,650 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=13, startTime=1731230717601; duration=0sec 2024-11-10T09:25:17,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/ffd9547f77b14fdd95631cbb9806a19e is 50, key is test_row_0/A:col10/1731230716653/Put/seqid=0 2024-11-10T09:25:17,650 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:17,651 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:17,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741894_1070 (size=14741) 2024-11-10T09:25:17,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/ffd9547f77b14fdd95631cbb9806a19e 2024-11-10T09:25:17,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/345f67aadbaf4227bf7fdfe8ce0f3d2d is 50, key is test_row_0/B:col10/1731230716653/Put/seqid=0 2024-11-10T09:25:17,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230777697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230777695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741895_1071 (size=12301) 2024-11-10T09:25:17,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/345f67aadbaf4227bf7fdfe8ce0f3d2d 2024-11-10T09:25:17,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/4c51ac2d7ee24842be7310c1864d0266 is 50, key is test_row_0/C:col10/1731230716653/Put/seqid=0 2024-11-10T09:25:17,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741896_1072 (size=12301) 2024-11-10T09:25:17,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:17,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230777806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230777815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230777815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230777846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:17,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:17,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:25:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230778016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230778026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230778026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,031 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:18,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:18,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:18,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:18,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:18,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:18,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/4c51ac2d7ee24842be7310c1864d0266 2024-11-10T09:25:18,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/ffd9547f77b14fdd95631cbb9806a19e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/ffd9547f77b14fdd95631cbb9806a19e 2024-11-10T09:25:18,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/ffd9547f77b14fdd95631cbb9806a19e, entries=200, sequenceid=292, filesize=14.4 K 2024-11-10T09:25:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/345f67aadbaf4227bf7fdfe8ce0f3d2d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/345f67aadbaf4227bf7fdfe8ce0f3d2d 2024-11-10T09:25:18,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/345f67aadbaf4227bf7fdfe8ce0f3d2d, entries=150, sequenceid=292, filesize=12.0 K 2024-11-10T09:25:18,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/4c51ac2d7ee24842be7310c1864d0266 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/4c51ac2d7ee24842be7310c1864d0266 2024-11-10T09:25:18,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/4c51ac2d7ee24842be7310c1864d0266, entries=150, sequenceid=292, filesize=12.0 K 2024-11-10T09:25:18,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 62a21c798fe3504a706a57ba0baaaff3 in 552ms, sequenceid=292, compaction requested=true 2024-11-10T09:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:18,182 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:18,182 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:18,184 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:18,184 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:18,184 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:18,184 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/75ac015e77e14875add11fd665366812, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4b2eecad80724be693626a133030b7ac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/ffd9547f77b14fdd95631cbb9806a19e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=41.2 K 2024-11-10T09:25:18,185 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:18,185 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75ac015e77e14875add11fd665366812, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230716092 2024-11-10T09:25:18,185 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:18,185 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:18,186 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c13987ce2fbf4270be331c6d4071abed, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3434beed70a04d0dbea25e28f39b7aea, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/345f67aadbaf4227bf7fdfe8ce0f3d2d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=36.5 K 2024-11-10T09:25:18,186 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2eecad80724be693626a133030b7ac, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230716609 2024-11-10T09:25:18,186 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c13987ce2fbf4270be331c6d4071abed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230716092 2024-11-10T09:25:18,186 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd9547f77b14fdd95631cbb9806a19e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230716653 2024-11-10T09:25:18,187 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3434beed70a04d0dbea25e28f39b7aea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230716609 2024-11-10T09:25:18,187 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 345f67aadbaf4227bf7fdfe8ce0f3d2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230716653 2024-11-10T09:25:18,201 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:18,202 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/cc9713ddee994cc7a9db47f891873e67 is 50, key is test_row_0/B:col10/1731230716653/Put/seqid=0 2024-11-10T09:25:18,206 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:18,207 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/1959e1e9e3534c1aba2dcedb92e080d9 is 50, key is test_row_0/A:col10/1731230716653/Put/seqid=0 2024-11-10T09:25:18,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741897_1073 (size=12983) 2024-11-10T09:25:18,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741898_1074 (size=12983) 2024-11-10T09:25:18,223 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/cc9713ddee994cc7a9db47f891873e67 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/cc9713ddee994cc7a9db47f891873e67 2024-11-10T09:25:18,231 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into cc9713ddee994cc7a9db47f891873e67(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:18,231 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:18,231 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=13, startTime=1731230718182; duration=0sec 2024-11-10T09:25:18,231 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:18,231 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:18,231 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T09:25:18,232 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T09:25:18,233 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T09:25:18,233 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. because compaction request was cancelled 2024-11-10T09:25:18,233 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:18,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T09:25:18,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:18,258 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:25:18,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/302eeaf894dc4b419ca72132de8e64f8 is 50, key is test_row_0/A:col10/1731230717691/Put/seqid=0 2024-11-10T09:25:18,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741899_1075 (size=12301) 2024-11-10T09:25:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:18,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:18,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230778351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230778354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230778466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230778466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,628 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/1959e1e9e3534c1aba2dcedb92e080d9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1959e1e9e3534c1aba2dcedb92e080d9 2024-11-10T09:25:18,636 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into 1959e1e9e3534c1aba2dcedb92e080d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:18,636 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:18,636 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=13, startTime=1731230718182; duration=0sec 2024-11-10T09:25:18,636 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:18,636 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:18,668 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/302eeaf894dc4b419ca72132de8e64f8 2024-11-10T09:25:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230778677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230778677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/66e23d7544cb451fb5008647e79509c0 is 50, key is test_row_0/B:col10/1731230717691/Put/seqid=0 2024-11-10T09:25:18,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741900_1076 (size=12301) 2024-11-10T09:25:18,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230778825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,828 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230778856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,857 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230778986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230778986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:25:19,089 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/66e23d7544cb451fb5008647e79509c0 2024-11-10T09:25:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/7dd05716ab52428faa057c557c86f5e6 is 50, key is test_row_0/C:col10/1731230717691/Put/seqid=0 2024-11-10T09:25:19,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741901_1077 (size=12301) 2024-11-10T09:25:19,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230779495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:19,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230779495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:19,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:19,511 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/7dd05716ab52428faa057c557c86f5e6 2024-11-10T09:25:19,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/302eeaf894dc4b419ca72132de8e64f8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/302eeaf894dc4b419ca72132de8e64f8 2024-11-10T09:25:19,523 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/302eeaf894dc4b419ca72132de8e64f8, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T09:25:19,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/66e23d7544cb451fb5008647e79509c0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/66e23d7544cb451fb5008647e79509c0 2024-11-10T09:25:19,532 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/66e23d7544cb451fb5008647e79509c0, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T09:25:19,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/7dd05716ab52428faa057c557c86f5e6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/7dd05716ab52428faa057c557c86f5e6 2024-11-10T09:25:19,539 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/7dd05716ab52428faa057c557c86f5e6, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T09:25:19,541 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 62a21c798fe3504a706a57ba0baaaff3 in 1283ms, sequenceid=317, compaction requested=true 2024-11-10T09:25:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T09:25:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T09:25:19,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T09:25:19,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6810 sec 2024-11-10T09:25:19,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.6900 sec 2024-11-10T09:25:20,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:20,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T09:25:20,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b79fe2ee537f4874a7b7ac4e716ea3b4 is 50, key is test_row_0/A:col10/1731230720057/Put/seqid=0 2024-11-10T09:25:20,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741902_1078 (size=14741) 2024-11-10T09:25:20,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230780139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230780246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230780456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b79fe2ee537f4874a7b7ac4e716ea3b4 2024-11-10T09:25:20,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3c4056945096467bb11bbfda52ea533c is 50, key is test_row_0/B:col10/1731230720057/Put/seqid=0 2024-11-10T09:25:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741903_1079 (size=12301) 2024-11-10T09:25:20,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230780506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230780506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,507 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,508 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230780766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230780835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,838 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230780866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:20,868 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:20,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3c4056945096467bb11bbfda52ea533c 2024-11-10T09:25:20,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/2e1980ffa95b40059d28110b54cba3c1 is 50, key is test_row_0/C:col10/1731230720057/Put/seqid=0 2024-11-10T09:25:20,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741904_1080 (size=12301) 2024-11-10T09:25:20,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/2e1980ffa95b40059d28110b54cba3c1 2024-11-10T09:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b79fe2ee537f4874a7b7ac4e716ea3b4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b79fe2ee537f4874a7b7ac4e716ea3b4 2024-11-10T09:25:20,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b79fe2ee537f4874a7b7ac4e716ea3b4, entries=200, sequenceid=332, filesize=14.4 K 2024-11-10T09:25:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/3c4056945096467bb11bbfda52ea533c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3c4056945096467bb11bbfda52ea533c 2024-11-10T09:25:20,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3c4056945096467bb11bbfda52ea533c, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T09:25:20,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/2e1980ffa95b40059d28110b54cba3c1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2e1980ffa95b40059d28110b54cba3c1 2024-11-10T09:25:20,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2e1980ffa95b40059d28110b54cba3c1, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T09:25:20,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 62a21c798fe3504a706a57ba0baaaff3 in 899ms, sequenceid=332, compaction requested=true 2024-11-10T09:25:20,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:20,958 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:20,958 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:20,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:20,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:20,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:20,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:20,960 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:20,960 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:20,960 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:20,960 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/cc9713ddee994cc7a9db47f891873e67, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/66e23d7544cb451fb5008647e79509c0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3c4056945096467bb11bbfda52ea533c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=36.7 K 2024-11-10T09:25:20,961 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:20,961 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:20,961 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:20,961 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1959e1e9e3534c1aba2dcedb92e080d9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/302eeaf894dc4b419ca72132de8e64f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b79fe2ee537f4874a7b7ac4e716ea3b4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=39.1 K 2024-11-10T09:25:20,961 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9713ddee994cc7a9db47f891873e67, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230716653 2024-11-10T09:25:20,962 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1959e1e9e3534c1aba2dcedb92e080d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230716653 2024-11-10T09:25:20,962 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e23d7544cb451fb5008647e79509c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731230717677 2024-11-10T09:25:20,963 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302eeaf894dc4b419ca72132de8e64f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731230717677 2024-11-10T09:25:20,963 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4056945096467bb11bbfda52ea533c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731230718343 2024-11-10T09:25:20,963 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting b79fe2ee537f4874a7b7ac4e716ea3b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731230718343 2024-11-10T09:25:20,977 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:20,978 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/f2b01dd8d006471a9ce8aa37fa64c072 is 50, key is test_row_0/A:col10/1731230720057/Put/seqid=0 2024-11-10T09:25:20,986 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:20,987 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/307d138375674728b9e6ae57b0005ad1 is 50, key is test_row_0/B:col10/1731230720057/Put/seqid=0 2024-11-10T09:25:20,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741905_1081 (size=13085) 2024-11-10T09:25:21,008 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/f2b01dd8d006471a9ce8aa37fa64c072 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f2b01dd8d006471a9ce8aa37fa64c072 2024-11-10T09:25:21,017 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into f2b01dd8d006471a9ce8aa37fa64c072(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:21,017 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:21,017 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=13, startTime=1731230720957; duration=0sec 2024-11-10T09:25:21,017 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:21,017 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:21,017 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:21,021 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:21,021 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:21,021 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:21,021 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/448e135a9633443282fd1b2bb49fc40c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/4c51ac2d7ee24842be7310c1864d0266, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/7dd05716ab52428faa057c557c86f5e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2e1980ffa95b40059d28110b54cba3c1] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=48.7 K 2024-11-10T09:25:21,022 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 448e135a9633443282fd1b2bb49fc40c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731230716609 2024-11-10T09:25:21,022 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c51ac2d7ee24842be7310c1864d0266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731230716653 2024-11-10T09:25:21,024 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dd05716ab52428faa057c557c86f5e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731230717677 2024-11-10T09:25:21,025 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e1980ffa95b40059d28110b54cba3c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731230718343 2024-11-10T09:25:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T09:25:21,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:21,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:21,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741906_1082 (size=13085) 2024-11-10T09:25:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T09:25:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:25:21,032 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:21,034 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:21,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:21,046 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/307d138375674728b9e6ae57b0005ad1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/307d138375674728b9e6ae57b0005ad1 2024-11-10T09:25:21,056 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:21,057 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9f130fa50f0545bb8c37a652e0c67d98 is 50, key is test_row_0/C:col10/1731230720057/Put/seqid=0 2024-11-10T09:25:21,058 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into 307d138375674728b9e6ae57b0005ad1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:21,058 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:21,058 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=13, startTime=1731230720958; duration=0sec 2024-11-10T09:25:21,058 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:21,058 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:21,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741907_1083 (size=13051) 2024-11-10T09:25:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:25:21,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T09:25:21,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:21,191 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:25:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/1d921ad6768b4f90bba760c4e9c16927 is 50, key is test_row_0/A:col10/1731230720125/Put/seqid=0 2024-11-10T09:25:21,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741908_1084 (size=12301) 2024-11-10T09:25:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:21,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:21,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230781313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:25:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230781426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:21,477 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9f130fa50f0545bb8c37a652e0c67d98 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9f130fa50f0545bb8c37a652e0c67d98 2024-11-10T09:25:21,485 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 9f130fa50f0545bb8c37a652e0c67d98(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:21,485 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:21,485 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=12, startTime=1731230720959; duration=0sec 2024-11-10T09:25:21,485 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:21,485 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:21,615 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/1d921ad6768b4f90bba760c4e9c16927 2024-11-10T09:25:21,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/e80a166b371945e4a5f0f058f4859686 is 50, key is test_row_0/B:col10/1731230720125/Put/seqid=0 2024-11-10T09:25:21,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741909_1085 (size=12301) 2024-11-10T09:25:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230781635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:25:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230781946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:22,028 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/e80a166b371945e4a5f0f058f4859686 2024-11-10T09:25:22,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8ea61e6451d743cbaaf519f5502b45e6 is 50, key is test_row_0/C:col10/1731230720125/Put/seqid=0 2024-11-10T09:25:22,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741910_1086 (size=12301) 2024-11-10T09:25:22,054 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8ea61e6451d743cbaaf519f5502b45e6 2024-11-10T09:25:22,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/1d921ad6768b4f90bba760c4e9c16927 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1d921ad6768b4f90bba760c4e9c16927 2024-11-10T09:25:22,069 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1d921ad6768b4f90bba760c4e9c16927, entries=150, sequenceid=356, filesize=12.0 K 2024-11-10T09:25:22,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/e80a166b371945e4a5f0f058f4859686 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e80a166b371945e4a5f0f058f4859686 2024-11-10T09:25:22,078 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e80a166b371945e4a5f0f058f4859686, entries=150, sequenceid=356, filesize=12.0 K 2024-11-10T09:25:22,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8ea61e6451d743cbaaf519f5502b45e6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8ea61e6451d743cbaaf519f5502b45e6 2024-11-10T09:25:22,092 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8ea61e6451d743cbaaf519f5502b45e6, entries=150, sequenceid=356, filesize=12.0 K 2024-11-10T09:25:22,093 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 62a21c798fe3504a706a57ba0baaaff3 in 902ms, sequenceid=356, compaction requested=false 2024-11-10T09:25:22,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:22,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:22,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T09:25:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T09:25:22,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T09:25:22,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0620 sec 2024-11-10T09:25:22,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0720 sec 2024-11-10T09:25:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T09:25:22,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:22,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T09:25:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:25:22,171 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:22,172 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:22,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:25:22,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T09:25:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:22,326 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T09:25:22,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/fbe490f556774beeb2b64d377a44ecca is 50, key is test_row_0/A:col10/1731230721307/Put/seqid=0 2024-11-10T09:25:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741911_1087 (size=12301) 2024-11-10T09:25:22,343 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/fbe490f556774beeb2b64d377a44ecca 2024-11-10T09:25:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/1e0d73aed17344669de00d2be999feaa is 50, key is test_row_0/B:col10/1731230721307/Put/seqid=0 2024-11-10T09:25:22,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741912_1088 (size=12301) 2024-11-10T09:25:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:22,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:25:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230782526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:22,527 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:22,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230782526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230782535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:22,537 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:22,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230782635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:22,773 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/1e0d73aed17344669de00d2be999feaa 2024-11-10T09:25:22,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/406fa1f368214e98b9155bd9cfb01584 is 50, key is test_row_0/C:col10/1731230721307/Put/seqid=0 2024-11-10T09:25:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:25:22,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741913_1089 (size=12301) 2024-11-10T09:25:22,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230782846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230783155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:23,201 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/406fa1f368214e98b9155bd9cfb01584 2024-11-10T09:25:23,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/fbe490f556774beeb2b64d377a44ecca as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/fbe490f556774beeb2b64d377a44ecca 2024-11-10T09:25:23,212 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/fbe490f556774beeb2b64d377a44ecca, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:25:23,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/1e0d73aed17344669de00d2be999feaa as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1e0d73aed17344669de00d2be999feaa 2024-11-10T09:25:23,218 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1e0d73aed17344669de00d2be999feaa, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:25:23,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/406fa1f368214e98b9155bd9cfb01584 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/406fa1f368214e98b9155bd9cfb01584 2024-11-10T09:25:23,224 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/406fa1f368214e98b9155bd9cfb01584, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:25:23,228 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 62a21c798fe3504a706a57ba0baaaff3 in 899ms, sequenceid=371, compaction requested=true 2024-11-10T09:25:23,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:23,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:23,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T09:25:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T09:25:23,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T09:25:23,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0580 sec 2024-11-10T09:25:23,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0640 sec 2024-11-10T09:25:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T09:25:23,306 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:23,306 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T09:25:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:25:23,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:23,310 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:23,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:25:23,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T09:25:23,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:23,464 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:25:23,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e587bd3e30434400af02ed31c87b2f2e is 50, key is test_row_0/A:col10/1731230722525/Put/seqid=0 2024-11-10T09:25:23,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741914_1090 (size=12301) 2024-11-10T09:25:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:25:23,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:23,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230783709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230783815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:23,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:23,875 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e587bd3e30434400af02ed31c87b2f2e 2024-11-10T09:25:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/ac78c025391e4b67ae2e7bac9ab80786 is 50, key is test_row_0/B:col10/1731230722525/Put/seqid=0 2024-11-10T09:25:23,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741915_1091 (size=12301) 2024-11-10T09:25:23,893 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/ac78c025391e4b67ae2e7bac9ab80786 2024-11-10T09:25:23,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/03c9494ee79a42dcbb0f1c950f3b6c1d is 50, key is test_row_0/C:col10/1731230722525/Put/seqid=0 2024-11-10T09:25:23,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741916_1092 (size=12301) 2024-11-10T09:25:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:25:24,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230784026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:24,308 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/03c9494ee79a42dcbb0f1c950f3b6c1d 2024-11-10T09:25:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/e587bd3e30434400af02ed31c87b2f2e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e587bd3e30434400af02ed31c87b2f2e 2024-11-10T09:25:24,321 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e587bd3e30434400af02ed31c87b2f2e, entries=150, sequenceid=393, filesize=12.0 K 2024-11-10T09:25:24,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/ac78c025391e4b67ae2e7bac9ab80786 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ac78c025391e4b67ae2e7bac9ab80786 2024-11-10T09:25:24,327 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ac78c025391e4b67ae2e7bac9ab80786, entries=150, sequenceid=393, filesize=12.0 K 2024-11-10T09:25:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/03c9494ee79a42dcbb0f1c950f3b6c1d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/03c9494ee79a42dcbb0f1c950f3b6c1d 2024-11-10T09:25:24,334 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/03c9494ee79a42dcbb0f1c950f3b6c1d, entries=150, sequenceid=393, filesize=12.0 K 2024-11-10T09:25:24,334 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 62a21c798fe3504a706a57ba0baaaff3 in 870ms, sequenceid=393, compaction requested=true 2024-11-10T09:25:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T09:25:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T09:25:24,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T09:25:24,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0260 sec 2024-11-10T09:25:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:24,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:24,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T09:25:24,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0330 sec 2024-11-10T09:25:24,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/6b9e5650d3114ba1ac0667102d9a79ae is 50, key is test_row_0/A:col10/1731230724337/Put/seqid=0 2024-11-10T09:25:24,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741917_1093 (size=14741) 2024-11-10T09:25:24,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/6b9e5650d3114ba1ac0667102d9a79ae 2024-11-10T09:25:24,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c57a361d24a14e4e9d969098100612a6 is 50, key is test_row_0/B:col10/1731230724337/Put/seqid=0 2024-11-10T09:25:24,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741918_1094 (size=12301) 2024-11-10T09:25:24,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c57a361d24a14e4e9d969098100612a6 2024-11-10T09:25:24,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/a225d5ebf5744e528c939c2471037ad3 is 50, key is test_row_0/C:col10/1731230724337/Put/seqid=0 2024-11-10T09:25:24,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741919_1095 (size=12301) 2024-11-10T09:25:24,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/a225d5ebf5744e528c939c2471037ad3 2024-11-10T09:25:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T09:25:24,446 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:24,447 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-10T09:25:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:25:24,450 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:24,452 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:24,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:24,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/6b9e5650d3114ba1ac0667102d9a79ae as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6b9e5650d3114ba1ac0667102d9a79ae 2024-11-10T09:25:24,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230784453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:24,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6b9e5650d3114ba1ac0667102d9a79ae, entries=200, sequenceid=408, filesize=14.4 K 2024-11-10T09:25:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c57a361d24a14e4e9d969098100612a6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c57a361d24a14e4e9d969098100612a6 2024-11-10T09:25:24,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c57a361d24a14e4e9d969098100612a6, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T09:25:24,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/a225d5ebf5744e528c939c2471037ad3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/a225d5ebf5744e528c939c2471037ad3 2024-11-10T09:25:24,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/a225d5ebf5744e528c939c2471037ad3, entries=150, sequenceid=408, filesize=12.0 K 2024-11-10T09:25:24,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 62a21c798fe3504a706a57ba0baaaff3 in 146ms, sequenceid=408, compaction requested=true 2024-11-10T09:25:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:24,485 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:25:24,485 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:25:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:24,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:24,487 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64729 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:25:24,487 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:25:24,487 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:24,487 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:24,487 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,487 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,487 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f2b01dd8d006471a9ce8aa37fa64c072, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1d921ad6768b4f90bba760c4e9c16927, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/fbe490f556774beeb2b64d377a44ecca, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e587bd3e30434400af02ed31c87b2f2e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6b9e5650d3114ba1ac0667102d9a79ae] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=63.2 K 2024-11-10T09:25:24,487 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/307d138375674728b9e6ae57b0005ad1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e80a166b371945e4a5f0f058f4859686, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1e0d73aed17344669de00d2be999feaa, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ac78c025391e4b67ae2e7bac9ab80786, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c57a361d24a14e4e9d969098100612a6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=60.8 K 2024-11-10T09:25:24,489 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 307d138375674728b9e6ae57b0005ad1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731230718343 2024-11-10T09:25:24,489 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b01dd8d006471a9ce8aa37fa64c072, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731230718343 2024-11-10T09:25:24,489 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d921ad6768b4f90bba760c4e9c16927, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731230720125 2024-11-10T09:25:24,489 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e80a166b371945e4a5f0f058f4859686, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731230720125 2024-11-10T09:25:24,490 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e0d73aed17344669de00d2be999feaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230721296 2024-11-10T09:25:24,490 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe490f556774beeb2b64d377a44ecca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230721296 2024-11-10T09:25:24,490 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ac78c025391e4b67ae2e7bac9ab80786, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230722498 2024-11-10T09:25:24,491 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e587bd3e30434400af02ed31c87b2f2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230722498 2024-11-10T09:25:24,491 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c57a361d24a14e4e9d969098100612a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230723703 2024-11-10T09:25:24,491 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b9e5650d3114ba1ac0667102d9a79ae, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230723698 2024-11-10T09:25:24,515 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:24,516 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c0c666c2f53747139e3505b999cd3f26 is 50, key is test_row_0/B:col10/1731230724337/Put/seqid=0 2024-11-10T09:25:24,529 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:24,529 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/5e7989c7ab644ee5898716bc261dc34d is 50, key is test_row_0/A:col10/1731230724337/Put/seqid=0 2024-11-10T09:25:24,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741920_1096 (size=13255) 2024-11-10T09:25:24,546 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/c0c666c2f53747139e3505b999cd3f26 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c0c666c2f53747139e3505b999cd3f26 2024-11-10T09:25:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:25:24,562 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into c0c666c2f53747139e3505b999cd3f26(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:24,562 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:24,563 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=11, startTime=1731230724485; duration=0sec 2024-11-10T09:25:24,564 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:24,564 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:24,564 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:25:24,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741921_1097 (size=13255) 2024-11-10T09:25:24,568 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:25:24,569 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:24,569 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,569 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9f130fa50f0545bb8c37a652e0c67d98, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8ea61e6451d743cbaaf519f5502b45e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/406fa1f368214e98b9155bd9cfb01584, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/03c9494ee79a42dcbb0f1c950f3b6c1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/a225d5ebf5744e528c939c2471037ad3] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=60.8 K 2024-11-10T09:25:24,570 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f130fa50f0545bb8c37a652e0c67d98, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731230718343 2024-11-10T09:25:24,572 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea61e6451d743cbaaf519f5502b45e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731230720125 2024-11-10T09:25:24,572 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 406fa1f368214e98b9155bd9cfb01584, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230721296 2024-11-10T09:25:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:24,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:25:24,573 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c9494ee79a42dcbb0f1c950f3b6c1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731230722498 2024-11-10T09:25:24,574 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a225d5ebf5744e528c939c2471037ad3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230723703 2024-11-10T09:25:24,576 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/5e7989c7ab644ee5898716bc261dc34d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5e7989c7ab644ee5898716bc261dc34d 2024-11-10T09:25:24,583 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into 5e7989c7ab644ee5898716bc261dc34d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:24,583 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:24,584 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=11, startTime=1731230724485; duration=0sec 2024-11-10T09:25:24,584 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:24,584 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:24,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/6a9bc46906014e36bd565d367ee411b3 is 50, key is test_row_0/A:col10/1731230724432/Put/seqid=0 2024-11-10T09:25:24,600 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#86 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:24,600 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0775408d743141ab891eb5b215370d43 is 50, key is test_row_0/C:col10/1731230724337/Put/seqid=0 2024-11-10T09:25:24,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T09:25:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741922_1098 (size=14741) 2024-11-10T09:25:24,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741923_1099 (size=13221) 2024-11-10T09:25:24,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230784620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:24,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230784735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:24,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:24,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T09:25:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:25:24,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50406 deadline: 1731230784876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:24,878 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:24,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50410 deadline: 1731230784886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:24,887 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:24,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T09:25:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:24,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:24,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230784946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:25,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/6a9bc46906014e36bd565d367ee411b3 2024-11-10T09:25:25,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/324fe3f902e84628aba24107835464d2 is 50, key is test_row_0/B:col10/1731230724432/Put/seqid=0 2024-11-10T09:25:25,035 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0775408d743141ab891eb5b215370d43 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0775408d743141ab891eb5b215370d43 2024-11-10T09:25:25,046 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 0775408d743141ab891eb5b215370d43(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:25,046 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:25,046 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=11, startTime=1731230724485; duration=0sec 2024-11-10T09:25:25,046 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:25,046 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:25,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741924_1100 (size=12301) 2024-11-10T09:25:25,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/324fe3f902e84628aba24107835464d2 2024-11-10T09:25:25,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T09:25:25,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:25,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:25,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:25,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:25,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:25,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/b0444671a1b24c4581ca6803c026fb2f is 50, key is test_row_0/C:col10/1731230724432/Put/seqid=0 2024-11-10T09:25:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:25:25,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741925_1101 (size=12301) 2024-11-10T09:25:25,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/b0444671a1b24c4581ca6803c026fb2f 2024-11-10T09:25:25,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/6a9bc46906014e36bd565d367ee411b3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6a9bc46906014e36bd565d367ee411b3 2024-11-10T09:25:25,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6a9bc46906014e36bd565d367ee411b3, entries=200, sequenceid=431, filesize=14.4 K 2024-11-10T09:25:25,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/324fe3f902e84628aba24107835464d2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/324fe3f902e84628aba24107835464d2 2024-11-10T09:25:25,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/324fe3f902e84628aba24107835464d2, entries=150, sequenceid=431, filesize=12.0 K 2024-11-10T09:25:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/b0444671a1b24c4581ca6803c026fb2f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/b0444671a1b24c4581ca6803c026fb2f 2024-11-10T09:25:25,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/b0444671a1b24c4581ca6803c026fb2f, entries=150, sequenceid=431, filesize=12.0 K 2024-11-10T09:25:25,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 62a21c798fe3504a706a57ba0baaaff3 in 563ms, sequenceid=431, compaction requested=false 2024-11-10T09:25:25,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:25,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T09:25:25,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:25,222 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T09:25:25,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/3ab24aef9f674dff8f7af536d9aa95b1 is 50, key is test_row_0/A:col10/1731230724612/Put/seqid=0 2024-11-10T09:25:25,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741926_1102 (size=12301) 2024-11-10T09:25:25,254 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/3ab24aef9f674dff8f7af536d9aa95b1 2024-11-10T09:25:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:25,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0bb0e99111a74be89d5afc53bcb4561c is 50, key is test_row_0/B:col10/1731230724612/Put/seqid=0 2024-11-10T09:25:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741927_1103 (size=12301) 2024-11-10T09:25:25,285 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0bb0e99111a74be89d5afc53bcb4561c 2024-11-10T09:25:25,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9081e17621e94c5e9aebc50baa6c5373 is 50, key is test_row_0/C:col10/1731230724612/Put/seqid=0 2024-11-10T09:25:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741928_1104 (size=12301) 2024-11-10T09:25:25,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230785346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230785456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:25:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230785666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:25,723 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9081e17621e94c5e9aebc50baa6c5373 2024-11-10T09:25:25,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/3ab24aef9f674dff8f7af536d9aa95b1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3ab24aef9f674dff8f7af536d9aa95b1 2024-11-10T09:25:25,734 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3ab24aef9f674dff8f7af536d9aa95b1, entries=150, sequenceid=447, filesize=12.0 K 2024-11-10T09:25:25,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0bb0e99111a74be89d5afc53bcb4561c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bb0e99111a74be89d5afc53bcb4561c 2024-11-10T09:25:25,744 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bb0e99111a74be89d5afc53bcb4561c, entries=150, sequenceid=447, filesize=12.0 K 2024-11-10T09:25:25,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/9081e17621e94c5e9aebc50baa6c5373 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9081e17621e94c5e9aebc50baa6c5373 2024-11-10T09:25:25,751 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9081e17621e94c5e9aebc50baa6c5373, entries=150, sequenceid=447, filesize=12.0 K 2024-11-10T09:25:25,752 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 62a21c798fe3504a706a57ba0baaaff3 in 531ms, sequenceid=447, compaction requested=true 2024-11-10T09:25:25,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-10T09:25:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-10T09:25:25,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T09:25:25,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-11-10T09:25:25,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3090 sec 2024-11-10T09:25:25,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:25,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:25,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:25:25,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/eb13995ee6754864a1a0a387b8b96bd8 is 50, key is test_row_0/A:col10/1731230725332/Put/seqid=0 2024-11-10T09:25:25,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741929_1105 (size=14741) 2024-11-10T09:25:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230786019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:26,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230786125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:26,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230786335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:26,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/eb13995ee6754864a1a0a387b8b96bd8 2024-11-10T09:25:26,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/dccde69d787a417d9105afa6cf3c47a2 is 50, key is test_row_0/B:col10/1731230725332/Put/seqid=0 2024-11-10T09:25:26,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741930_1106 (size=12301) 2024-11-10T09:25:26,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50374 deadline: 1731230786536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:26,537 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:26,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50372 deadline: 1731230786575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:26,578 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 62a21c798fe3504a706a57ba0baaaff3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T09:25:26,596 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:26,596 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-10T09:25:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:25:26,598 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:26,599 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:26,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230786646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:25:26,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T09:25:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:26,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:26,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:26,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/dccde69d787a417d9105afa6cf3c47a2 2024-11-10T09:25:26,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0c38b7e079d84027afcfaa57c7355144 is 50, key is test_row_0/C:col10/1731230725332/Put/seqid=0 2024-11-10T09:25:26,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741931_1107 (size=12301) 2024-11-10T09:25:26,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T09:25:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:26,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:25:27,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T09:25:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230787155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:27,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T09:25:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:27,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:27,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0c38b7e079d84027afcfaa57c7355144 2024-11-10T09:25:27,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/eb13995ee6754864a1a0a387b8b96bd8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/eb13995ee6754864a1a0a387b8b96bd8 2024-11-10T09:25:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:25:27,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/eb13995ee6754864a1a0a387b8b96bd8, entries=200, sequenceid=470, filesize=14.4 K 2024-11-10T09:25:27,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/dccde69d787a417d9105afa6cf3c47a2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/dccde69d787a417d9105afa6cf3c47a2 2024-11-10T09:25:27,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/dccde69d787a417d9105afa6cf3c47a2, entries=150, sequenceid=470, filesize=12.0 K 2024-11-10T09:25:27,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/0c38b7e079d84027afcfaa57c7355144 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c38b7e079d84027afcfaa57c7355144 2024-11-10T09:25:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c38b7e079d84027afcfaa57c7355144, entries=150, sequenceid=470, filesize=12.0 K 2024-11-10T09:25:27,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 62a21c798fe3504a706a57ba0baaaff3 in 1270ms, sequenceid=470, compaction requested=true 2024-11-10T09:25:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:27,247 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:27,247 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:27,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:27,249 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:27,249 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:27,249 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:27,249 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:27,249 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,249 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,249 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5e7989c7ab644ee5898716bc261dc34d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6a9bc46906014e36bd565d367ee411b3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3ab24aef9f674dff8f7af536d9aa95b1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/eb13995ee6754864a1a0a387b8b96bd8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=53.7 K 2024-11-10T09:25:27,249 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c0c666c2f53747139e3505b999cd3f26, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/324fe3f902e84628aba24107835464d2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bb0e99111a74be89d5afc53bcb4561c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/dccde69d787a417d9105afa6cf3c47a2] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=49.0 K 2024-11-10T09:25:27,249 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e7989c7ab644ee5898716bc261dc34d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230723703 2024-11-10T09:25:27,249 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c666c2f53747139e3505b999cd3f26, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230723703 2024-11-10T09:25:27,250 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a9bc46906014e36bd565d367ee411b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731230724426 2024-11-10T09:25:27,250 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 324fe3f902e84628aba24107835464d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731230724432 2024-11-10T09:25:27,250 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ab24aef9f674dff8f7af536d9aa95b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731230724605 2024-11-10T09:25:27,250 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb0e99111a74be89d5afc53bcb4561c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731230724605 2024-11-10T09:25:27,250 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb13995ee6754864a1a0a387b8b96bd8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731230725332 2024-11-10T09:25:27,250 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting dccde69d787a417d9105afa6cf3c47a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731230725332 2024-11-10T09:25:27,266 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:27,266 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/81e977b91eda4707a52f2d2d2dd05008 is 50, key is test_row_0/B:col10/1731230725332/Put/seqid=0 2024-11-10T09:25:27,267 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:27,268 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/5219c0e23d8f4fe6a37e0a1812f918a8 is 50, key is test_row_0/A:col10/1731230725332/Put/seqid=0 2024-11-10T09:25:27,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741932_1108 (size=13391) 2024-11-10T09:25:27,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741933_1109 (size=13391) 2024-11-10T09:25:27,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T09:25:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,363 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:25:27,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/60be934f34374e62af416b7799628990 is 50, key is test_row_0/A:col10/1731230726014/Put/seqid=0 2024-11-10T09:25:27,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741934_1110 (size=12301) 2024-11-10T09:25:27,684 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/81e977b91eda4707a52f2d2d2dd05008 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81e977b91eda4707a52f2d2d2dd05008 2024-11-10T09:25:27,688 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/5219c0e23d8f4fe6a37e0a1812f918a8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5219c0e23d8f4fe6a37e0a1812f918a8 2024-11-10T09:25:27,692 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into 81e977b91eda4707a52f2d2d2dd05008(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:27,692 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:27,692 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=12, startTime=1731230727247; duration=0sec 2024-11-10T09:25:27,692 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:27,692 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:27,692 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:25:27,694 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:25:27,694 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:27,694 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:27,694 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0775408d743141ab891eb5b215370d43, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/b0444671a1b24c4581ca6803c026fb2f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9081e17621e94c5e9aebc50baa6c5373, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c38b7e079d84027afcfaa57c7355144] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=48.9 K 2024-11-10T09:25:27,694 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into 5219c0e23d8f4fe6a37e0a1812f918a8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:27,694 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:27,694 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=12, startTime=1731230727247; duration=0sec 2024-11-10T09:25:27,694 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0775408d743141ab891eb5b215370d43, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731230723703 2024-11-10T09:25:27,694 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:27,694 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:27,695 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting b0444671a1b24c4581ca6803c026fb2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731230724432 2024-11-10T09:25:27,695 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 9081e17621e94c5e9aebc50baa6c5373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1731230724605 2024-11-10T09:25:27,695 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c38b7e079d84027afcfaa57c7355144, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731230725332 2024-11-10T09:25:27,705 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:27,706 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c309e8f7b46443fdae14a2282fd22ac9 is 50, key is test_row_0/C:col10/1731230725332/Put/seqid=0 2024-11-10T09:25:27,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741935_1111 (size=13357) 2024-11-10T09:25:27,716 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c309e8f7b46443fdae14a2282fd22ac9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c309e8f7b46443fdae14a2282fd22ac9 2024-11-10T09:25:27,722 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into c309e8f7b46443fdae14a2282fd22ac9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:27,722 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:27,722 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=12, startTime=1731230727247; duration=0sec 2024-11-10T09:25:27,722 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:27,722 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:25:27,774 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/60be934f34374e62af416b7799628990 2024-11-10T09:25:27,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fb3684ec54c1475d9e34ce6285e2fca5 is 50, key is test_row_0/B:col10/1731230726014/Put/seqid=0 2024-11-10T09:25:27,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741936_1112 (size=12301) 2024-11-10T09:25:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:28,190 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fb3684ec54c1475d9e34ce6285e2fca5 2024-11-10T09:25:28,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c1fee863fe1143aa9c53db0b6f47b083 is 50, key is test_row_0/C:col10/1731230726014/Put/seqid=0 2024-11-10T09:25:28,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741937_1113 (size=12301) 2024-11-10T09:25:28,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230788236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230788345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:28,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230788556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:28,603 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c1fee863fe1143aa9c53db0b6f47b083 2024-11-10T09:25:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/60be934f34374e62af416b7799628990 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/60be934f34374e62af416b7799628990 2024-11-10T09:25:28,613 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/60be934f34374e62af416b7799628990, entries=150, sequenceid=483, filesize=12.0 K 2024-11-10T09:25:28,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fb3684ec54c1475d9e34ce6285e2fca5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fb3684ec54c1475d9e34ce6285e2fca5 2024-11-10T09:25:28,619 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fb3684ec54c1475d9e34ce6285e2fca5, entries=150, sequenceid=483, filesize=12.0 K 2024-11-10T09:25:28,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/c1fee863fe1143aa9c53db0b6f47b083 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c1fee863fe1143aa9c53db0b6f47b083 2024-11-10T09:25:28,629 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c1fee863fe1143aa9c53db0b6f47b083, entries=150, sequenceid=483, filesize=12.0 K 2024-11-10T09:25:28,630 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 62a21c798fe3504a706a57ba0baaaff3 in 1267ms, sequenceid=483, compaction requested=false 2024-11-10T09:25:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-10T09:25:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-10T09:25:28,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T09:25:28,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0330 sec 2024-11-10T09:25:28,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0380 sec 2024-11-10T09:25:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T09:25:28,746 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:28,746 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-10T09:25:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T09:25:28,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:28,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:28,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T09:25:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:28,867 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:28,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:25:28,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/91e3c95d0c9643e0a2c67ed3255edd44 is 50, key is test_row_0/A:col10/1731230728225/Put/seqid=0 2024-11-10T09:25:28,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741938_1114 (size=14741) 2024-11-10T09:25:28,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/91e3c95d0c9643e0a2c67ed3255edd44 2024-11-10T09:25:28,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fe61e613e0124001b1b9a49e7abc6590 is 50, key is test_row_0/B:col10/1731230728225/Put/seqid=0 2024-11-10T09:25:28,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741939_1115 (size=12301) 2024-11-10T09:25:28,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fe61e613e0124001b1b9a49e7abc6590 2024-11-10T09:25:28,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230788897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:28,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8b760b57fac04590943ffad519be0745 is 50, key is test_row_0/C:col10/1731230728225/Put/seqid=0 2024-11-10T09:25:28,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T09:25:28,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:28,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:28,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:28,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:28,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:28,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741940_1116 (size=12301) 2024-11-10T09:25:29,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230789005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:29,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T09:25:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T09:25:29,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T09:25:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:29,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230789215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:29,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8b760b57fac04590943ffad519be0745 2024-11-10T09:25:29,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/91e3c95d0c9643e0a2c67ed3255edd44 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/91e3c95d0c9643e0a2c67ed3255edd44 2024-11-10T09:25:29,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/91e3c95d0c9643e0a2c67ed3255edd44, entries=200, sequenceid=510, filesize=14.4 K 2024-11-10T09:25:29,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fe61e613e0124001b1b9a49e7abc6590 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fe61e613e0124001b1b9a49e7abc6590 2024-11-10T09:25:29,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fe61e613e0124001b1b9a49e7abc6590, entries=150, sequenceid=510, filesize=12.0 K 2024-11-10T09:25:29,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/8b760b57fac04590943ffad519be0745 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8b760b57fac04590943ffad519be0745 2024-11-10T09:25:29,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8b760b57fac04590943ffad519be0745, entries=150, sequenceid=510, filesize=12.0 K 2024-11-10T09:25:29,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 62a21c798fe3504a706a57ba0baaaff3 in 472ms, sequenceid=510, compaction requested=true 2024-11-10T09:25:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:29,340 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:29,340 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:29,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:29,341 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:29,341 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:29,341 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,341 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5219c0e23d8f4fe6a37e0a1812f918a8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/60be934f34374e62af416b7799628990, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/91e3c95d0c9643e0a2c67ed3255edd44] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=39.5 K 2024-11-10T09:25:29,342 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:29,342 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:29,342 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,342 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81e977b91eda4707a52f2d2d2dd05008, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fb3684ec54c1475d9e34ce6285e2fca5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fe61e613e0124001b1b9a49e7abc6590] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=37.1 K 2024-11-10T09:25:29,342 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e977b91eda4707a52f2d2d2dd05008, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731230725332 2024-11-10T09:25:29,343 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5219c0e23d8f4fe6a37e0a1812f918a8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731230725332 2024-11-10T09:25:29,343 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting fb3684ec54c1475d9e34ce6285e2fca5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731230726009 2024-11-10T09:25:29,344 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60be934f34374e62af416b7799628990, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731230726009 2024-11-10T09:25:29,344 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting fe61e613e0124001b1b9a49e7abc6590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731230728216 2024-11-10T09:25:29,344 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91e3c95d0c9643e0a2c67ed3255edd44, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731230728216 2024-11-10T09:25:29,354 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:29,355 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/0bce19548e1c4dfca2d33d4f6fec2c74 is 50, key is test_row_0/A:col10/1731230728225/Put/seqid=0 2024-11-10T09:25:29,358 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:29,359 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fc476789220d4bc7ab321269cbecca00 is 50, key is test_row_0/B:col10/1731230728225/Put/seqid=0 2024-11-10T09:25:29,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T09:25:29,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,362 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:25:29,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741941_1117 (size=13493) 2024-11-10T09:25:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T09:25:29,380 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/0bce19548e1c4dfca2d33d4f6fec2c74 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/0bce19548e1c4dfca2d33d4f6fec2c74 2024-11-10T09:25:29,391 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into 0bce19548e1c4dfca2d33d4f6fec2c74(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:29,391 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:29,392 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=13, startTime=1731230729340; duration=0sec 2024-11-10T09:25:29,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/af22dce5876b42929505c9ce8abc1f1d is 50, key is test_row_0/A:col10/1731230728895/Put/seqid=0 2024-11-10T09:25:29,393 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:29,393 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:29,393 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:29,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741942_1118 (size=13493) 2024-11-10T09:25:29,397 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:29,397 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:29,397 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:29,397 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c309e8f7b46443fdae14a2282fd22ac9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c1fee863fe1143aa9c53db0b6f47b083, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8b760b57fac04590943ffad519be0745] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=37.1 K 2024-11-10T09:25:29,398 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c309e8f7b46443fdae14a2282fd22ac9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731230725332 2024-11-10T09:25:29,399 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1fee863fe1143aa9c53db0b6f47b083, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731230726009 2024-11-10T09:25:29,399 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b760b57fac04590943ffad519be0745, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731230728216 2024-11-10T09:25:29,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741943_1119 (size=12301) 2024-11-10T09:25:29,430 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:29,431 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/ba445ba127014df7b63b6e695efb617b is 50, key is test_row_0/C:col10/1731230728225/Put/seqid=0 2024-11-10T09:25:29,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741944_1120 (size=13459) 2024-11-10T09:25:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:29,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. as already flushing 2024-11-10T09:25:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230789617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230789725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:29,801 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/fc476789220d4bc7ab321269cbecca00 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fc476789220d4bc7ab321269cbecca00 2024-11-10T09:25:29,808 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into fc476789220d4bc7ab321269cbecca00(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:29,808 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:29,808 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=13, startTime=1731230729340; duration=0sec 2024-11-10T09:25:29,808 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:29,808 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:29,819 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/af22dce5876b42929505c9ce8abc1f1d 2024-11-10T09:25:29,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/1416cce9440f4075bdfa11fd3c4eec9d is 50, key is test_row_0/B:col10/1731230728895/Put/seqid=0 2024-11-10T09:25:29,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741945_1121 (size=12301) 2024-11-10T09:25:29,853 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/ba445ba127014df7b63b6e695efb617b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/ba445ba127014df7b63b6e695efb617b 2024-11-10T09:25:29,859 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into ba445ba127014df7b63b6e695efb617b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:29,859 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:29,859 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=13, startTime=1731230729340; duration=0sec 2024-11-10T09:25:29,860 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:29,860 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T09:25:29,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230789935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:30,231 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/1416cce9440f4075bdfa11fd3c4eec9d 2024-11-10T09:25:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/3bd85c9f03de4711ab378b731addf63d is 50, key is test_row_0/C:col10/1731230728895/Put/seqid=0 2024-11-10T09:25:30,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741946_1122 (size=12301) 2024-11-10T09:25:30,242 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/3bd85c9f03de4711ab378b731addf63d 2024-11-10T09:25:30,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/af22dce5876b42929505c9ce8abc1f1d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/af22dce5876b42929505c9ce8abc1f1d 2024-11-10T09:25:30,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50398 deadline: 1731230790246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:30,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:25:30,252 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/af22dce5876b42929505c9ce8abc1f1d, entries=150, sequenceid=522, filesize=12.0 K 2024-11-10T09:25:30,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/1416cce9440f4075bdfa11fd3c4eec9d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1416cce9440f4075bdfa11fd3c4eec9d 2024-11-10T09:25:30,258 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1416cce9440f4075bdfa11fd3c4eec9d, entries=150, sequenceid=522, filesize=12.0 K 2024-11-10T09:25:30,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/3bd85c9f03de4711ab378b731addf63d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/3bd85c9f03de4711ab378b731addf63d 2024-11-10T09:25:30,265 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/3bd85c9f03de4711ab378b731addf63d, entries=150, sequenceid=522, filesize=12.0 K 2024-11-10T09:25:30,266 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 62a21c798fe3504a706a57ba0baaaff3 in 904ms, sequenceid=522, compaction requested=false 2024-11-10T09:25:30,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:30,267 INFO [Thread-155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-155. 2024-11-10T09:25:30,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:30,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-10T09:25:30,267 DEBUG [Thread-155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-10T09:25:30,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T09:25:30,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5180 sec 2024-11-10T09:25:30,272 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.5240 sec 2024-11-10T09:25:30,273 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:30,275 INFO [Thread-157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-157. 2024-11-10T09:25:30,275 DEBUG [Thread-157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:30,275 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,275 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:30,276 INFO [Thread-159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-159. 2024-11-10T09:25:30,276 DEBUG [Thread-159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:30,276 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,276 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:30,278 INFO [Thread-161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-161. 2024-11-10T09:25:30,278 DEBUG [Thread-161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:30,278 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,279 DEBUG [Thread-161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:30,760 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:30,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:25:30,760 INFO [Thread-152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-152. 2024-11-10T09:25:30,761 DEBUG [Thread-152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:30,761 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,761 DEBUG [Thread-152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:30,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:30,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b86e50d977ed4632be0f5b98e5a3a231 is 50, key is test_row_0/A:col10/1731230729615/Put/seqid=0 2024-11-10T09:25:30,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741947_1123 (size=12301) 2024-11-10T09:25:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T09:25:30,896 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:31,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b86e50d977ed4632be0f5b98e5a3a231 2024-11-10T09:25:31,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0399a1ccd0e2418bb64bf909f1a22818 is 50, key is test_row_0/B:col10/1731230729615/Put/seqid=0 2024-11-10T09:25:31,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741948_1124 (size=12301) 2024-11-10T09:25:31,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0399a1ccd0e2418bb64bf909f1a22818 2024-11-10T09:25:31,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/5b7aae42c06545d3a54b2b63f1da25c8 is 50, key is test_row_0/C:col10/1731230729615/Put/seqid=0 2024-11-10T09:25:31,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741949_1125 (size=12301) 2024-11-10T09:25:32,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/5b7aae42c06545d3a54b2b63f1da25c8 2024-11-10T09:25:32,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/b86e50d977ed4632be0f5b98e5a3a231 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b86e50d977ed4632be0f5b98e5a3a231 2024-11-10T09:25:32,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b86e50d977ed4632be0f5b98e5a3a231, entries=150, sequenceid=550, filesize=12.0 K 2024-11-10T09:25:32,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/0399a1ccd0e2418bb64bf909f1a22818 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0399a1ccd0e2418bb64bf909f1a22818 2024-11-10T09:25:32,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0399a1ccd0e2418bb64bf909f1a22818, entries=150, sequenceid=550, filesize=12.0 K 2024-11-10T09:25:32,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/5b7aae42c06545d3a54b2b63f1da25c8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5b7aae42c06545d3a54b2b63f1da25c8 2024-11-10T09:25:32,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5b7aae42c06545d3a54b2b63f1da25c8, entries=150, sequenceid=550, filesize=12.0 K 2024-11-10T09:25:32,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 62a21c798fe3504a706a57ba0baaaff3 in 1287ms, sequenceid=550, compaction requested=true 2024-11-10T09:25:32,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:32,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:32,048 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:32,048 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 62a21c798fe3504a706a57ba0baaaff3:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:32,049 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:32,049 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:32,049 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/B is initiating minor compaction (all files) 2024-11-10T09:25:32,049 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/A is initiating minor compaction (all files) 2024-11-10T09:25:32,049 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/B in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:32,049 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/A in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:32,049 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fc476789220d4bc7ab321269cbecca00, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1416cce9440f4075bdfa11fd3c4eec9d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0399a1ccd0e2418bb64bf909f1a22818] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=37.2 K 2024-11-10T09:25:32,049 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/0bce19548e1c4dfca2d33d4f6fec2c74, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/af22dce5876b42929505c9ce8abc1f1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b86e50d977ed4632be0f5b98e5a3a231] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=37.2 K 2024-11-10T09:25:32,050 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting fc476789220d4bc7ab321269cbecca00, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731230728216 2024-11-10T09:25:32,050 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bce19548e1c4dfca2d33d4f6fec2c74, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731230728216 2024-11-10T09:25:32,050 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 1416cce9440f4075bdfa11fd3c4eec9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731230728881 2024-11-10T09:25:32,050 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting af22dce5876b42929505c9ce8abc1f1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731230728881 2024-11-10T09:25:32,050 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0399a1ccd0e2418bb64bf909f1a22818, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731230729594 2024-11-10T09:25:32,051 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting b86e50d977ed4632be0f5b98e5a3a231, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731230729594 2024-11-10T09:25:32,059 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#B#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:32,060 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#A#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:32,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/57214e14d91144d48a29733b4a74b9d5 is 50, key is test_row_0/B:col10/1731230729615/Put/seqid=0 2024-11-10T09:25:32,060 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/1ada9a234ed7437b87e3f02c79602923 is 50, key is test_row_0/A:col10/1731230729615/Put/seqid=0 2024-11-10T09:25:32,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741950_1126 (size=13595) 2024-11-10T09:25:32,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741951_1127 (size=13595) 2024-11-10T09:25:32,070 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/1ada9a234ed7437b87e3f02c79602923 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1ada9a234ed7437b87e3f02c79602923 2024-11-10T09:25:32,075 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/A of 62a21c798fe3504a706a57ba0baaaff3 into 1ada9a234ed7437b87e3f02c79602923(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:32,075 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:32,075 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/A, priority=13, startTime=1731230732047; duration=0sec 2024-11-10T09:25:32,076 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:32,076 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:A 2024-11-10T09:25:32,076 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:32,077 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:32,077 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 62a21c798fe3504a706a57ba0baaaff3/C is initiating minor compaction (all files) 2024-11-10T09:25:32,077 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 62a21c798fe3504a706a57ba0baaaff3/C in TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:32,077 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/ba445ba127014df7b63b6e695efb617b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/3bd85c9f03de4711ab378b731addf63d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5b7aae42c06545d3a54b2b63f1da25c8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp, totalSize=37.2 K 2024-11-10T09:25:32,077 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba445ba127014df7b63b6e695efb617b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731230728216 2024-11-10T09:25:32,078 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd85c9f03de4711ab378b731addf63d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=522, earliestPutTs=1731230728881 2024-11-10T09:25:32,078 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b7aae42c06545d3a54b2b63f1da25c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731230729594 2024-11-10T09:25:32,085 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 62a21c798fe3504a706a57ba0baaaff3#C#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:32,086 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/208b977499be459fbddff7bf18ac957d is 50, key is test_row_0/C:col10/1731230729615/Put/seqid=0 2024-11-10T09:25:32,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741952_1128 (size=13561) 2024-11-10T09:25:32,145 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:25:32,478 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/57214e14d91144d48a29733b4a74b9d5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/57214e14d91144d48a29733b4a74b9d5 2024-11-10T09:25:32,487 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/B of 62a21c798fe3504a706a57ba0baaaff3 into 57214e14d91144d48a29733b4a74b9d5(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:32,487 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:32,487 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/B, priority=13, startTime=1731230732048; duration=0sec 2024-11-10T09:25:32,487 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:32,487 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:B 2024-11-10T09:25:32,495 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/208b977499be459fbddff7bf18ac957d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/208b977499be459fbddff7bf18ac957d 2024-11-10T09:25:32,501 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 62a21c798fe3504a706a57ba0baaaff3/C of 62a21c798fe3504a706a57ba0baaaff3 into 208b977499be459fbddff7bf18ac957d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:32,501 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 62a21c798fe3504a706a57ba0baaaff3: 2024-11-10T09:25:32,501 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., storeName=62a21c798fe3504a706a57ba0baaaff3/C, priority=13, startTime=1731230732048; duration=0sec 2024-11-10T09:25:32,501 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:32,501 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 62a21c798fe3504a706a57ba0baaaff3:C 2024-11-10T09:25:34,910 INFO [Thread-146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-146. 2024-11-10T09:25:34,911 DEBUG [Thread-146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:34,911 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:34,911 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:34,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:34,989 INFO [Thread-148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-148. 2024-11-10T09:25:34,989 DEBUG [Thread-148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:34,989 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:34,989 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:34,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:36,629 INFO [Thread-144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-144. 2024-11-10T09:25:36,629 DEBUG [Thread-144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:36,629 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:36,629 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:36,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:36,639 INFO [Thread-150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-150. 2024-11-10T09:25:36,639 DEBUG [Thread-150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:25:36,639 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:36,639 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:36,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:36,640 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 207 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3727 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3722 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:25:36,641 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1690 2024-11-10T09:25:36,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5067 rows 2024-11-10T09:25:36,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1691 2024-11-10T09:25:36,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5067 rows 2024-11-10T09:25:36,642 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:25:36,642 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:25:36,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:36,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:36,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:25:36,649 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:25:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:25:36,656 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230736656"}]},"ts":"1731230736656"} 2024-11-10T09:25:36,657 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:25:36,658 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:25:36,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:25:36,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, UNASSIGN}] 2024-11-10T09:25:36,665 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, UNASSIGN 2024-11-10T09:25:36,666 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=62a21c798fe3504a706a57ba0baaaff3, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:36,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, UNASSIGN because future has completed 2024-11-10T09:25:36,668 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:25:36,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:25:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:25:36,826 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:36,827 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:25:36,828 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 62a21c798fe3504a706a57ba0baaaff3, disabling compactions & flushes 2024-11-10T09:25:36,828 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:36,828 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:36,828 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. after waiting 0 ms 2024-11-10T09:25:36,828 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:36,828 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 62a21c798fe3504a706a57ba0baaaff3 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T09:25:36,835 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/3da8d42b682d4448862c8637f7e65ffc is 50, key is test_row_1/A:col10/1731230736636/Put/seqid=0 2024-11-10T09:25:36,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741953_1129 (size=9857) 2024-11-10T09:25:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:25:37,240 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/3da8d42b682d4448862c8637f7e65ffc 2024-11-10T09:25:37,257 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/b753a2e30e8f43009047e4604c07213c is 50, key is test_row_1/B:col10/1731230736636/Put/seqid=0 2024-11-10T09:25:37,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741954_1130 (size=9857) 2024-11-10T09:25:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:25:37,661 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/b753a2e30e8f43009047e4604c07213c 2024-11-10T09:25:37,670 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/1a60bc3b8f7b4188bd493d59e733cfcf is 50, key is test_row_1/C:col10/1731230736636/Put/seqid=0 2024-11-10T09:25:37,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741955_1131 (size=9857) 2024-11-10T09:25:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:25:38,076 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/1a60bc3b8f7b4188bd493d59e733cfcf 2024-11-10T09:25:38,088 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/A/3da8d42b682d4448862c8637f7e65ffc as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3da8d42b682d4448862c8637f7e65ffc 2024-11-10T09:25:38,094 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3da8d42b682d4448862c8637f7e65ffc, entries=100, sequenceid=560, filesize=9.6 K 2024-11-10T09:25:38,095 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/B/b753a2e30e8f43009047e4604c07213c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/b753a2e30e8f43009047e4604c07213c 2024-11-10T09:25:38,099 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/b753a2e30e8f43009047e4604c07213c, entries=100, sequenceid=560, filesize=9.6 K 2024-11-10T09:25:38,100 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/.tmp/C/1a60bc3b8f7b4188bd493d59e733cfcf as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/1a60bc3b8f7b4188bd493d59e733cfcf 2024-11-10T09:25:38,104 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/1a60bc3b8f7b4188bd493d59e733cfcf, entries=100, sequenceid=560, filesize=9.6 K 2024-11-10T09:25:38,105 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 62a21c798fe3504a706a57ba0baaaff3 in 1277ms, sequenceid=560, compaction requested=false 2024-11-10T09:25:38,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e79e8d504998498ebb07b58320f558b1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/8cbfaf1722e54e338443592a726293f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f4170317d521420a941addf6eb8f68ac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/d94ee52b31ae48d5846b2d7b87dd2cf3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b443cbf5c1974412b2ef8b33824d32c1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c491ca41bf7b49779a41c93fe862d018, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/43db33b024c7493f8394f3c499797d1a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/86388ce2631a41ad9f1c9afdb2866a5e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4ce583dc9c114855a710b2594bb699f3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4a1a618f22bc4db88b3b224950ba7545, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e03e713283b548f09e78e96f6b08b670, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f87f31a4b9944390a4dce26e0a194d95, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e92b76ec704b41268e29b4b396b1dd86, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/aad0c17c62ea4c5ca302c6dbfc813a2d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bd916235d93e4a75bcf8fbb3aa287a59, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c38004d50f124174a168387b8bd3a8e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bc5e894fc4c64c16a8155c69520a4c31, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/75ac015e77e14875add11fd665366812, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4b2eecad80724be693626a133030b7ac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/ffd9547f77b14fdd95631cbb9806a19e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1959e1e9e3534c1aba2dcedb92e080d9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/302eeaf894dc4b419ca72132de8e64f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b79fe2ee537f4874a7b7ac4e716ea3b4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f2b01dd8d006471a9ce8aa37fa64c072, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1d921ad6768b4f90bba760c4e9c16927, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/fbe490f556774beeb2b64d377a44ecca, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e587bd3e30434400af02ed31c87b2f2e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6b9e5650d3114ba1ac0667102d9a79ae, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5e7989c7ab644ee5898716bc261dc34d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6a9bc46906014e36bd565d367ee411b3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3ab24aef9f674dff8f7af536d9aa95b1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/eb13995ee6754864a1a0a387b8b96bd8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5219c0e23d8f4fe6a37e0a1812f918a8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/60be934f34374e62af416b7799628990, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/91e3c95d0c9643e0a2c67ed3255edd44, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/0bce19548e1c4dfca2d33d4f6fec2c74, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/af22dce5876b42929505c9ce8abc1f1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b86e50d977ed4632be0f5b98e5a3a231] to archive 2024-11-10T09:25:38,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:25:38,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e79e8d504998498ebb07b58320f558b1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e79e8d504998498ebb07b58320f558b1 2024-11-10T09:25:38,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/8cbfaf1722e54e338443592a726293f8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/8cbfaf1722e54e338443592a726293f8 2024-11-10T09:25:38,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f4170317d521420a941addf6eb8f68ac to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f4170317d521420a941addf6eb8f68ac 2024-11-10T09:25:38,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/d94ee52b31ae48d5846b2d7b87dd2cf3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/d94ee52b31ae48d5846b2d7b87dd2cf3 2024-11-10T09:25:38,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b443cbf5c1974412b2ef8b33824d32c1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b443cbf5c1974412b2ef8b33824d32c1 2024-11-10T09:25:38,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c491ca41bf7b49779a41c93fe862d018 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c491ca41bf7b49779a41c93fe862d018 2024-11-10T09:25:38,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/43db33b024c7493f8394f3c499797d1a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/43db33b024c7493f8394f3c499797d1a 2024-11-10T09:25:38,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/86388ce2631a41ad9f1c9afdb2866a5e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/86388ce2631a41ad9f1c9afdb2866a5e 2024-11-10T09:25:38,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4ce583dc9c114855a710b2594bb699f3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4ce583dc9c114855a710b2594bb699f3 2024-11-10T09:25:38,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4a1a618f22bc4db88b3b224950ba7545 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4a1a618f22bc4db88b3b224950ba7545 2024-11-10T09:25:38,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e03e713283b548f09e78e96f6b08b670 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e03e713283b548f09e78e96f6b08b670 2024-11-10T09:25:38,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f87f31a4b9944390a4dce26e0a194d95 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f87f31a4b9944390a4dce26e0a194d95 2024-11-10T09:25:38,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e92b76ec704b41268e29b4b396b1dd86 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e92b76ec704b41268e29b4b396b1dd86 2024-11-10T09:25:38,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/aad0c17c62ea4c5ca302c6dbfc813a2d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/aad0c17c62ea4c5ca302c6dbfc813a2d 2024-11-10T09:25:38,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bd916235d93e4a75bcf8fbb3aa287a59 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bd916235d93e4a75bcf8fbb3aa287a59 2024-11-10T09:25:38,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c38004d50f124174a168387b8bd3a8e6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/c38004d50f124174a168387b8bd3a8e6 2024-11-10T09:25:38,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bc5e894fc4c64c16a8155c69520a4c31 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/bc5e894fc4c64c16a8155c69520a4c31 2024-11-10T09:25:38,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/75ac015e77e14875add11fd665366812 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/75ac015e77e14875add11fd665366812 2024-11-10T09:25:38,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4b2eecad80724be693626a133030b7ac to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/4b2eecad80724be693626a133030b7ac 2024-11-10T09:25:38,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/ffd9547f77b14fdd95631cbb9806a19e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/ffd9547f77b14fdd95631cbb9806a19e 2024-11-10T09:25:38,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1959e1e9e3534c1aba2dcedb92e080d9 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1959e1e9e3534c1aba2dcedb92e080d9 2024-11-10T09:25:38,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/302eeaf894dc4b419ca72132de8e64f8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/302eeaf894dc4b419ca72132de8e64f8 2024-11-10T09:25:38,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b79fe2ee537f4874a7b7ac4e716ea3b4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b79fe2ee537f4874a7b7ac4e716ea3b4 2024-11-10T09:25:38,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f2b01dd8d006471a9ce8aa37fa64c072 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/f2b01dd8d006471a9ce8aa37fa64c072 2024-11-10T09:25:38,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1d921ad6768b4f90bba760c4e9c16927 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1d921ad6768b4f90bba760c4e9c16927 2024-11-10T09:25:38,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/fbe490f556774beeb2b64d377a44ecca to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/fbe490f556774beeb2b64d377a44ecca 2024-11-10T09:25:38,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e587bd3e30434400af02ed31c87b2f2e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/e587bd3e30434400af02ed31c87b2f2e 2024-11-10T09:25:38,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6b9e5650d3114ba1ac0667102d9a79ae to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6b9e5650d3114ba1ac0667102d9a79ae 2024-11-10T09:25:38,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5e7989c7ab644ee5898716bc261dc34d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5e7989c7ab644ee5898716bc261dc34d 2024-11-10T09:25:38,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6a9bc46906014e36bd565d367ee411b3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/6a9bc46906014e36bd565d367ee411b3 2024-11-10T09:25:38,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3ab24aef9f674dff8f7af536d9aa95b1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3ab24aef9f674dff8f7af536d9aa95b1 2024-11-10T09:25:38,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/eb13995ee6754864a1a0a387b8b96bd8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/eb13995ee6754864a1a0a387b8b96bd8 2024-11-10T09:25:38,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5219c0e23d8f4fe6a37e0a1812f918a8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/5219c0e23d8f4fe6a37e0a1812f918a8 2024-11-10T09:25:38,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/60be934f34374e62af416b7799628990 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/60be934f34374e62af416b7799628990 2024-11-10T09:25:38,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/91e3c95d0c9643e0a2c67ed3255edd44 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/91e3c95d0c9643e0a2c67ed3255edd44 2024-11-10T09:25:38,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/0bce19548e1c4dfca2d33d4f6fec2c74 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/0bce19548e1c4dfca2d33d4f6fec2c74 2024-11-10T09:25:38,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/af22dce5876b42929505c9ce8abc1f1d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/af22dce5876b42929505c9ce8abc1f1d 2024-11-10T09:25:38,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b86e50d977ed4632be0f5b98e5a3a231 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/b86e50d977ed4632be0f5b98e5a3a231 2024-11-10T09:25:38,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ef496a1a80064361b0ceac504626b3a0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/94dfa4e719144950afbb74ba0f5d5e12, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81ee1b964aa0435882abccc50322989c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c60bf496e6034be89cc7c3b11f25e83e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70bcc9077d3e429da15fbc1b7d4e2de5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bdd9fd551b841eab14f7390c1bc34c5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70dff667220f40019cb8da6f4c7ce63b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/608f811fd361465eba1f6302cc9b486c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/d64dd032cd0c40599619ec4cf1838c24, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/810ac48bf3cc41d88a307e6ea58cda1b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/2faf6d57c7bd4efd841e63c60700de5d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/598705ac1c5849ca9311fed6138a5202, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/192ed45510c249b9a1e30e6749ef1172, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/7870eea5e20c416586323074de8de536, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e9817331c00b41f3b295db72dfc7db5b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/585733090e174c2bb8d659097ff48ff3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c13987ce2fbf4270be331c6d4071abed, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3d60c06f1e884eef8364c58b56c0cfb5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3434beed70a04d0dbea25e28f39b7aea, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/cc9713ddee994cc7a9db47f891873e67, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/345f67aadbaf4227bf7fdfe8ce0f3d2d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/66e23d7544cb451fb5008647e79509c0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/307d138375674728b9e6ae57b0005ad1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3c4056945096467bb11bbfda52ea533c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e80a166b371945e4a5f0f058f4859686, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1e0d73aed17344669de00d2be999feaa, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ac78c025391e4b67ae2e7bac9ab80786, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c0c666c2f53747139e3505b999cd3f26, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c57a361d24a14e4e9d969098100612a6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/324fe3f902e84628aba24107835464d2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bb0e99111a74be89d5afc53bcb4561c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81e977b91eda4707a52f2d2d2dd05008, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/dccde69d787a417d9105afa6cf3c47a2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fb3684ec54c1475d9e34ce6285e2fca5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fc476789220d4bc7ab321269cbecca00, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fe61e613e0124001b1b9a49e7abc6590, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1416cce9440f4075bdfa11fd3c4eec9d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0399a1ccd0e2418bb64bf909f1a22818] to archive 2024-11-10T09:25:38,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:25:38,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ef496a1a80064361b0ceac504626b3a0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ef496a1a80064361b0ceac504626b3a0 2024-11-10T09:25:38,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/94dfa4e719144950afbb74ba0f5d5e12 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/94dfa4e719144950afbb74ba0f5d5e12 2024-11-10T09:25:38,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81ee1b964aa0435882abccc50322989c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81ee1b964aa0435882abccc50322989c 2024-11-10T09:25:38,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c60bf496e6034be89cc7c3b11f25e83e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c60bf496e6034be89cc7c3b11f25e83e 2024-11-10T09:25:38,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70bcc9077d3e429da15fbc1b7d4e2de5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70bcc9077d3e429da15fbc1b7d4e2de5 2024-11-10T09:25:38,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bdd9fd551b841eab14f7390c1bc34c5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bdd9fd551b841eab14f7390c1bc34c5 2024-11-10T09:25:38,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70dff667220f40019cb8da6f4c7ce63b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/70dff667220f40019cb8da6f4c7ce63b 2024-11-10T09:25:38,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/608f811fd361465eba1f6302cc9b486c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/608f811fd361465eba1f6302cc9b486c 2024-11-10T09:25:38,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/d64dd032cd0c40599619ec4cf1838c24 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/d64dd032cd0c40599619ec4cf1838c24 2024-11-10T09:25:38,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/810ac48bf3cc41d88a307e6ea58cda1b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/810ac48bf3cc41d88a307e6ea58cda1b 2024-11-10T09:25:38,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/2faf6d57c7bd4efd841e63c60700de5d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/2faf6d57c7bd4efd841e63c60700de5d 2024-11-10T09:25:38,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/598705ac1c5849ca9311fed6138a5202 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/598705ac1c5849ca9311fed6138a5202 2024-11-10T09:25:38,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/192ed45510c249b9a1e30e6749ef1172 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/192ed45510c249b9a1e30e6749ef1172 2024-11-10T09:25:38,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/7870eea5e20c416586323074de8de536 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/7870eea5e20c416586323074de8de536 2024-11-10T09:25:38,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e9817331c00b41f3b295db72dfc7db5b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e9817331c00b41f3b295db72dfc7db5b 2024-11-10T09:25:38,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/585733090e174c2bb8d659097ff48ff3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/585733090e174c2bb8d659097ff48ff3 2024-11-10T09:25:38,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c13987ce2fbf4270be331c6d4071abed to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c13987ce2fbf4270be331c6d4071abed 2024-11-10T09:25:38,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3d60c06f1e884eef8364c58b56c0cfb5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3d60c06f1e884eef8364c58b56c0cfb5 2024-11-10T09:25:38,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3434beed70a04d0dbea25e28f39b7aea to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3434beed70a04d0dbea25e28f39b7aea 2024-11-10T09:25:38,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/cc9713ddee994cc7a9db47f891873e67 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/cc9713ddee994cc7a9db47f891873e67 2024-11-10T09:25:38,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/345f67aadbaf4227bf7fdfe8ce0f3d2d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/345f67aadbaf4227bf7fdfe8ce0f3d2d 2024-11-10T09:25:38,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/66e23d7544cb451fb5008647e79509c0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/66e23d7544cb451fb5008647e79509c0 2024-11-10T09:25:38,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/307d138375674728b9e6ae57b0005ad1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/307d138375674728b9e6ae57b0005ad1 2024-11-10T09:25:38,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3c4056945096467bb11bbfda52ea533c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/3c4056945096467bb11bbfda52ea533c 2024-11-10T09:25:38,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e80a166b371945e4a5f0f058f4859686 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/e80a166b371945e4a5f0f058f4859686 2024-11-10T09:25:38,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1e0d73aed17344669de00d2be999feaa to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1e0d73aed17344669de00d2be999feaa 2024-11-10T09:25:38,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ac78c025391e4b67ae2e7bac9ab80786 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/ac78c025391e4b67ae2e7bac9ab80786 2024-11-10T09:25:38,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c0c666c2f53747139e3505b999cd3f26 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c0c666c2f53747139e3505b999cd3f26 2024-11-10T09:25:38,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c57a361d24a14e4e9d969098100612a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/c57a361d24a14e4e9d969098100612a6 2024-11-10T09:25:38,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/324fe3f902e84628aba24107835464d2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/324fe3f902e84628aba24107835464d2 2024-11-10T09:25:38,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bb0e99111a74be89d5afc53bcb4561c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0bb0e99111a74be89d5afc53bcb4561c 2024-11-10T09:25:38,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81e977b91eda4707a52f2d2d2dd05008 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/81e977b91eda4707a52f2d2d2dd05008 2024-11-10T09:25:38,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/dccde69d787a417d9105afa6cf3c47a2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/dccde69d787a417d9105afa6cf3c47a2 2024-11-10T09:25:38,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fb3684ec54c1475d9e34ce6285e2fca5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fb3684ec54c1475d9e34ce6285e2fca5 2024-11-10T09:25:38,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fc476789220d4bc7ab321269cbecca00 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fc476789220d4bc7ab321269cbecca00 2024-11-10T09:25:38,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fe61e613e0124001b1b9a49e7abc6590 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/fe61e613e0124001b1b9a49e7abc6590 2024-11-10T09:25:38,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1416cce9440f4075bdfa11fd3c4eec9d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/1416cce9440f4075bdfa11fd3c4eec9d 2024-11-10T09:25:38,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0399a1ccd0e2418bb64bf909f1a22818 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/0399a1ccd0e2418bb64bf909f1a22818 2024-11-10T09:25:38,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9413ebcd82d146b4a673e1b5e1f66b6b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8cb9d60230c7488dacece929683ef7cd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2a32f6b5fcbb4df99062c9e718314435, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/06ae6e752034449bac3d24e82ed3a93f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/48e3d47380914521a759f362c3339fee, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/377796ce6ef84bebbf422cba12c28ddc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/67732838438a4aaabc91e3bb5b10208c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5afcea01cc374dee91b011184bb1d06b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0db94f1fb7d34a17b78ecf040283aa84, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c1f18be4f11461f9a1d229076cbce5c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/40efca6cf57e4e8b915f21637507125f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/665641f3b40242c38232b9da95012fc8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9c2065d2f9e34c5880a3167fe955cec3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/f2ca59cf07684fea8422d31b11d8b2a8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/606c2b87e4074d5596af728df49c5340, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/babb3899565e45f9a772f24093f52563, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c87877506192451aa679a12491f257af, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/448e135a9633443282fd1b2bb49fc40c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/d9045ea43c2a4aeba788db5ce87d180a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/4c51ac2d7ee24842be7310c1864d0266, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/7dd05716ab52428faa057c557c86f5e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9f130fa50f0545bb8c37a652e0c67d98, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2e1980ffa95b40059d28110b54cba3c1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8ea61e6451d743cbaaf519f5502b45e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/406fa1f368214e98b9155bd9cfb01584, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/03c9494ee79a42dcbb0f1c950f3b6c1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0775408d743141ab891eb5b215370d43, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/a225d5ebf5744e528c939c2471037ad3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/b0444671a1b24c4581ca6803c026fb2f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9081e17621e94c5e9aebc50baa6c5373, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c309e8f7b46443fdae14a2282fd22ac9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c38b7e079d84027afcfaa57c7355144, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c1fee863fe1143aa9c53db0b6f47b083, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/ba445ba127014df7b63b6e695efb617b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8b760b57fac04590943ffad519be0745, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/3bd85c9f03de4711ab378b731addf63d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5b7aae42c06545d3a54b2b63f1da25c8] to archive 2024-11-10T09:25:38,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:25:38,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9413ebcd82d146b4a673e1b5e1f66b6b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9413ebcd82d146b4a673e1b5e1f66b6b 2024-11-10T09:25:38,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8cb9d60230c7488dacece929683ef7cd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8cb9d60230c7488dacece929683ef7cd 2024-11-10T09:25:38,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2a32f6b5fcbb4df99062c9e718314435 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2a32f6b5fcbb4df99062c9e718314435 2024-11-10T09:25:38,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/06ae6e752034449bac3d24e82ed3a93f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/06ae6e752034449bac3d24e82ed3a93f 2024-11-10T09:25:38,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/48e3d47380914521a759f362c3339fee to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/48e3d47380914521a759f362c3339fee 2024-11-10T09:25:38,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/377796ce6ef84bebbf422cba12c28ddc to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/377796ce6ef84bebbf422cba12c28ddc 2024-11-10T09:25:38,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/67732838438a4aaabc91e3bb5b10208c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/67732838438a4aaabc91e3bb5b10208c 2024-11-10T09:25:38,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5afcea01cc374dee91b011184bb1d06b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5afcea01cc374dee91b011184bb1d06b 2024-11-10T09:25:38,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0db94f1fb7d34a17b78ecf040283aa84 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0db94f1fb7d34a17b78ecf040283aa84 2024-11-10T09:25:38,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c1f18be4f11461f9a1d229076cbce5c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c1f18be4f11461f9a1d229076cbce5c 2024-11-10T09:25:38,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/40efca6cf57e4e8b915f21637507125f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/40efca6cf57e4e8b915f21637507125f 2024-11-10T09:25:38,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/665641f3b40242c38232b9da95012fc8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/665641f3b40242c38232b9da95012fc8 2024-11-10T09:25:38,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9c2065d2f9e34c5880a3167fe955cec3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9c2065d2f9e34c5880a3167fe955cec3 2024-11-10T09:25:38,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/f2ca59cf07684fea8422d31b11d8b2a8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/f2ca59cf07684fea8422d31b11d8b2a8 2024-11-10T09:25:38,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/606c2b87e4074d5596af728df49c5340 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/606c2b87e4074d5596af728df49c5340 2024-11-10T09:25:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/babb3899565e45f9a772f24093f52563 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/babb3899565e45f9a772f24093f52563 2024-11-10T09:25:38,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c87877506192451aa679a12491f257af to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c87877506192451aa679a12491f257af 2024-11-10T09:25:38,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/448e135a9633443282fd1b2bb49fc40c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/448e135a9633443282fd1b2bb49fc40c 2024-11-10T09:25:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/d9045ea43c2a4aeba788db5ce87d180a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/d9045ea43c2a4aeba788db5ce87d180a 2024-11-10T09:25:38,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/4c51ac2d7ee24842be7310c1864d0266 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/4c51ac2d7ee24842be7310c1864d0266 2024-11-10T09:25:38,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/7dd05716ab52428faa057c557c86f5e6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/7dd05716ab52428faa057c557c86f5e6 2024-11-10T09:25:38,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9f130fa50f0545bb8c37a652e0c67d98 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9f130fa50f0545bb8c37a652e0c67d98 2024-11-10T09:25:38,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2e1980ffa95b40059d28110b54cba3c1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/2e1980ffa95b40059d28110b54cba3c1 2024-11-10T09:25:38,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8ea61e6451d743cbaaf519f5502b45e6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8ea61e6451d743cbaaf519f5502b45e6 2024-11-10T09:25:38,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/406fa1f368214e98b9155bd9cfb01584 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/406fa1f368214e98b9155bd9cfb01584 2024-11-10T09:25:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/03c9494ee79a42dcbb0f1c950f3b6c1d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/03c9494ee79a42dcbb0f1c950f3b6c1d 2024-11-10T09:25:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0775408d743141ab891eb5b215370d43 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0775408d743141ab891eb5b215370d43 2024-11-10T09:25:38,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/a225d5ebf5744e528c939c2471037ad3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/a225d5ebf5744e528c939c2471037ad3 2024-11-10T09:25:38,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/b0444671a1b24c4581ca6803c026fb2f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/b0444671a1b24c4581ca6803c026fb2f 2024-11-10T09:25:38,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9081e17621e94c5e9aebc50baa6c5373 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/9081e17621e94c5e9aebc50baa6c5373 2024-11-10T09:25:38,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c309e8f7b46443fdae14a2282fd22ac9 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c309e8f7b46443fdae14a2282fd22ac9 2024-11-10T09:25:38,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c38b7e079d84027afcfaa57c7355144 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/0c38b7e079d84027afcfaa57c7355144 2024-11-10T09:25:38,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c1fee863fe1143aa9c53db0b6f47b083 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/c1fee863fe1143aa9c53db0b6f47b083 2024-11-10T09:25:38,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/ba445ba127014df7b63b6e695efb617b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/ba445ba127014df7b63b6e695efb617b 2024-11-10T09:25:38,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8b760b57fac04590943ffad519be0745 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/8b760b57fac04590943ffad519be0745 2024-11-10T09:25:38,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/3bd85c9f03de4711ab378b731addf63d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/3bd85c9f03de4711ab378b731addf63d 2024-11-10T09:25:38,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5b7aae42c06545d3a54b2b63f1da25c8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/5b7aae42c06545d3a54b2b63f1da25c8 2024-11-10T09:25:38,255 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/recovered.edits/563.seqid, newMaxSeqId=563, maxSeqId=1 2024-11-10T09:25:38,255 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. 2024-11-10T09:25:38,256 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 62a21c798fe3504a706a57ba0baaaff3: Waiting for close lock at 1731230736828Running coprocessor pre-close hooks at 1731230736828Disabling compacts and flushes for region at 1731230736828Disabling writes for close at 1731230736828Obtaining lock to block concurrent updates at 1731230736828Preparing flush snapshotting stores in 62a21c798fe3504a706a57ba0baaaff3 at 1731230736828Finished memstore snapshotting TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731230736829 (+1 ms)Flushing stores of TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3. at 1731230736830 (+1 ms)Flushing 62a21c798fe3504a706a57ba0baaaff3/A: creating writer at 1731230736830Flushing 62a21c798fe3504a706a57ba0baaaff3/A: appending metadata at 1731230736834 (+4 ms)Flushing 62a21c798fe3504a706a57ba0baaaff3/A: closing flushed file at 1731230736834Flushing 62a21c798fe3504a706a57ba0baaaff3/B: creating writer at 1731230737251 (+417 ms)Flushing 62a21c798fe3504a706a57ba0baaaff3/B: appending metadata at 1731230737256 (+5 ms)Flushing 62a21c798fe3504a706a57ba0baaaff3/B: closing flushed file at 1731230737256Flushing 62a21c798fe3504a706a57ba0baaaff3/C: creating writer at 1731230737666 (+410 ms)Flushing 62a21c798fe3504a706a57ba0baaaff3/C: appending metadata at 1731230737669 (+3 ms)Flushing 62a21c798fe3504a706a57ba0baaaff3/C: closing flushed file at 1731230737670 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b1a6ca2: reopening flushed file at 1731230738087 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e7a9eaf: reopening flushed file at 1731230738094 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10cad453: reopening flushed file at 1731230738099 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 62a21c798fe3504a706a57ba0baaaff3 in 1277ms, sequenceid=560, compaction requested=false at 1731230738105 (+6 ms)Writing region close event to WAL at 1731230738251 (+146 ms)Running coprocessor post-close hooks at 1731230738255 (+4 ms)Closed at 1731230738255 2024-11-10T09:25:38,258 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:38,258 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=62a21c798fe3504a706a57ba0baaaff3, regionState=CLOSED 2024-11-10T09:25:38,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:25:38,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T09:25:38,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 62a21c798fe3504a706a57ba0baaaff3, server=802346a334f5,32883,1731230704436 in 1.5930 sec 2024-11-10T09:25:38,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-10T09:25:38,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=62a21c798fe3504a706a57ba0baaaff3, UNASSIGN in 1.5990 sec 2024-11-10T09:25:38,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-10T09:25:38,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6070 sec 2024-11-10T09:25:38,269 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230738268"}]},"ts":"1731230738268"} 2024-11-10T09:25:38,270 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:25:38,270 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:25:38,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6200 sec 2024-11-10T09:25:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T09:25:38,806 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:38,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:25:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:38,813 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:38,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-10T09:25:38,814 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:38,820 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:38,824 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/recovered.edits] 2024-11-10T09:25:38,827 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1ada9a234ed7437b87e3f02c79602923 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/1ada9a234ed7437b87e3f02c79602923 2024-11-10T09:25:38,828 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3da8d42b682d4448862c8637f7e65ffc to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/A/3da8d42b682d4448862c8637f7e65ffc 2024-11-10T09:25:38,830 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/57214e14d91144d48a29733b4a74b9d5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/57214e14d91144d48a29733b4a74b9d5 2024-11-10T09:25:38,831 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/b753a2e30e8f43009047e4604c07213c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/B/b753a2e30e8f43009047e4604c07213c 2024-11-10T09:25:38,833 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/1a60bc3b8f7b4188bd493d59e733cfcf to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/1a60bc3b8f7b4188bd493d59e733cfcf 2024-11-10T09:25:38,834 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/208b977499be459fbddff7bf18ac957d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/C/208b977499be459fbddff7bf18ac957d 2024-11-10T09:25:38,837 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/recovered.edits/563.seqid to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3/recovered.edits/563.seqid 2024-11-10T09:25:38,838 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/62a21c798fe3504a706a57ba0baaaff3 2024-11-10T09:25:38,838 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:25:38,842 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:38,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T09:25:38,850 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:25:38,875 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:25:38,876 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:38,876 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:25:38,877 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230738876"}]},"ts":"9223372036854775807"} 2024-11-10T09:25:38,881 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:25:38,881 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 62a21c798fe3504a706a57ba0baaaff3, NAME => 'TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:25:38,881 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:25:38,881 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230738881"}]},"ts":"9223372036854775807"} 2024-11-10T09:25:38,884 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:25:38,884 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:38,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 75 msec 2024-11-10T09:25:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-10T09:25:38,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:25:38,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:38,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@20be0a11[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;802346a334f5:32883-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1008509712_22 at /127.0.0.1:38394 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=552 (was 666), ProcessCount=11 (was 11), AvailableMemoryMB=7066 (was 7793) 2024-11-10T09:25:38,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=552, ProcessCount=11, AvailableMemoryMB=7065 2024-11-10T09:25:38,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:25:38,946 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:25:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:38,948 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:25:38,948 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:38,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-10T09:25:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T09:25:38,949 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:25:38,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741956_1132 (size=959) 2024-11-10T09:25:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T09:25:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T09:25:39,359 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:25:39,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741957_1133 (size=53) 2024-11-10T09:25:39,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T09:25:39,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:39,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5619dfa0d7c1c633ebe4b8aecdc69e58, disabling compactions & flushes 2024-11-10T09:25:39,770 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:39,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:39,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. after waiting 0 ms 2024-11-10T09:25:39,770 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:39,770 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:39,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: Waiting for close lock at 1731230739770Disabling compacts and flushes for region at 1731230739770Disabling writes for close at 1731230739770Writing region close event to WAL at 1731230739770Closed at 1731230739770 2024-11-10T09:25:39,773 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:25:39,774 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230739773"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230739773"}]},"ts":"1731230739773"} 2024-11-10T09:25:39,778 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:25:39,780 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:25:39,780 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230739780"}]},"ts":"1731230739780"} 2024-11-10T09:25:39,782 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:25:39,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, ASSIGN}] 2024-11-10T09:25:39,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, ASSIGN 2024-11-10T09:25:39,785 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, ASSIGN; state=OFFLINE, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=false 2024-11-10T09:25:39,936 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:39,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, ASSIGN because future has completed 2024-11-10T09:25:39,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T09:25:40,146 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:40,147 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:25:40,147 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,147 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:40,148 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,148 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,151 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,153 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5619dfa0d7c1c633ebe4b8aecdc69e58 columnFamilyName A 2024-11-10T09:25:40,154 DEBUG [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:40,154 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(327): Store=5619dfa0d7c1c633ebe4b8aecdc69e58/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:40,154 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,156 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5619dfa0d7c1c633ebe4b8aecdc69e58 columnFamilyName B 2024-11-10T09:25:40,156 DEBUG [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:40,156 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(327): Store=5619dfa0d7c1c633ebe4b8aecdc69e58/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:40,157 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,158 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5619dfa0d7c1c633ebe4b8aecdc69e58 columnFamilyName C 2024-11-10T09:25:40,158 DEBUG [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:40,159 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(327): Store=5619dfa0d7c1c633ebe4b8aecdc69e58/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:40,159 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,160 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,160 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,161 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,161 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,162 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:25:40,163 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,165 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:25:40,166 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 5619dfa0d7c1c633ebe4b8aecdc69e58; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67236370, jitterRate=0.0018999874591827393}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:25:40,166 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:40,166 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: Running coprocessor pre-open hook at 1731230740148Writing region info on filesystem at 1731230740148Initializing all the Stores at 1731230740150 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230740150Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230740150Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230740150Cleaning up temporary data from old regions at 1731230740161 (+11 ms)Running coprocessor post-open hooks at 1731230740166 (+5 ms)Region opened successfully at 1731230740166 2024-11-10T09:25:40,167 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., pid=36, masterSystemTime=1731230740139 2024-11-10T09:25:40,169 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:40,169 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:40,169 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:40,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:25:40,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-10T09:25:40,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 in 229 msec 2024-11-10T09:25:40,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-10T09:25:40,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, ASSIGN in 390 msec 2024-11-10T09:25:40,176 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:25:40,176 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230740176"}]},"ts":"1731230740176"} 2024-11-10T09:25:40,177 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:25:40,178 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:25:40,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2320 sec 2024-11-10T09:25:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T09:25:41,096 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:41,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e30321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:41,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:41,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:41,099 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a751cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:41,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:41,102 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df92b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:41,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:41,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:41,108 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,38235,1731230703274 2024-11-10T09:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@34f193ca 2024-11-10T09:25:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:25:41,110 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:25:41,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:25:41,117 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:25:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T09:25:41,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741958_1134 (size=995) 2024-11-10T09:25:41,557 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T09:25:41,557 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T09:25:41,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:25:41,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, REOPEN/MOVE}] 2024-11-10T09:25:41,572 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, REOPEN/MOVE 2024-11-10T09:25:41,573 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:41,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, REOPEN/MOVE because future has completed 2024-11-10T09:25:41,575 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:25:41,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:25:41,728 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:41,729 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:25:41,729 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 5619dfa0d7c1c633ebe4b8aecdc69e58, disabling compactions & flushes 2024-11-10T09:25:41,729 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:41,729 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:41,729 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. after waiting 0 ms 2024-11-10T09:25:41,729 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:41,738 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T09:25:41,739 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:41,739 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: Waiting for close lock at 1731230741729Running coprocessor pre-close hooks at 1731230741729Disabling compacts and flushes for region at 1731230741729Disabling writes for close at 1731230741729Writing region close event to WAL at 1731230741730 (+1 ms)Running coprocessor post-close hooks at 1731230741739 (+9 ms)Closed at 1731230741739 2024-11-10T09:25:41,739 WARN [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 5619dfa0d7c1c633ebe4b8aecdc69e58 to self. 2024-11-10T09:25:41,741 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:41,741 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=CLOSED 2024-11-10T09:25:41,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:25:41,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-10T09:25:41,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 in 169 msec 2024-11-10T09:25:41,747 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, REOPEN/MOVE; state=CLOSED, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=true 2024-11-10T09:25:41,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:41,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, REOPEN/MOVE because future has completed 2024-11-10T09:25:41,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:25:42,065 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,065 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:25:42,066 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,066 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:25:42,066 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,067 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,074 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,080 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5619dfa0d7c1c633ebe4b8aecdc69e58 columnFamilyName A 2024-11-10T09:25:42,083 DEBUG [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:42,083 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(327): Store=5619dfa0d7c1c633ebe4b8aecdc69e58/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:42,084 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,084 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5619dfa0d7c1c633ebe4b8aecdc69e58 columnFamilyName B 2024-11-10T09:25:42,085 DEBUG [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:42,085 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(327): Store=5619dfa0d7c1c633ebe4b8aecdc69e58/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:42,085 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,086 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5619dfa0d7c1c633ebe4b8aecdc69e58 columnFamilyName C 2024-11-10T09:25:42,086 DEBUG [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:42,087 INFO [StoreOpener-5619dfa0d7c1c633ebe4b8aecdc69e58-1 {}] regionserver.HStore(327): Store=5619dfa0d7c1c633ebe4b8aecdc69e58/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:25:42,087 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,087 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,088 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,089 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,089 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,090 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:25:42,091 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,092 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 5619dfa0d7c1c633ebe4b8aecdc69e58; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64133514, jitterRate=-0.044336169958114624}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:25:42,092 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,094 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: Running coprocessor pre-open hook at 1731230742067Writing region info on filesystem at 1731230742067Initializing all the Stores at 1731230742068 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230742069 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230742069Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230742069Cleaning up temporary data from old regions at 1731230742089 (+20 ms)Running coprocessor post-open hooks at 1731230742092 (+3 ms)Region opened successfully at 1731230742094 (+2 ms) 2024-11-10T09:25:42,095 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., pid=41, masterSystemTime=1731230742058 2024-11-10T09:25:42,097 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,097 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,097 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=OPEN, openSeqNum=5, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:25:42,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-10T09:25:42,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 in 197 msec 2024-11-10T09:25:42,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T09:25:42,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, REOPEN/MOVE in 531 msec 2024-11-10T09:25:42,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-10T09:25:42,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 542 msec 2024-11-10T09:25:42,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 987 msec 2024-11-10T09:25:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-10T09:25:42,112 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:42,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,113 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa96732, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,115 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,116 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4f64ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,116 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,116 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,117 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b08cfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,118 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b3c879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5c81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,121 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@350b5793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,122 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266719ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,125 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8e4ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,126 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db37eb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,128 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7e18e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,130 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228ce28d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,132 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@146d1eaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,133 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f8d325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a14e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,135 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e84579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,137 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604865ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17c234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,139 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c438738, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,140 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@630d23ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197f7df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:25:42,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:25:42,142 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:25:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:25:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:25:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7afa3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:25:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:25:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:25:42,143 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:25:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1a2f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:25:42,145 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:42,145 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,145 DEBUG [Thread-669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,145 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,145 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,146 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,146 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T09:25:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,146 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,146 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,147 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:25:42,148 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:42,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:25:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,150 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:42,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,154 DEBUG [Thread-669 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:25:42,155 DEBUG [Thread-669 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:25:42,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,157 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:25:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:25:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,206 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:42,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:25:42,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107137a8ab98844c5aa075fe914ac563aa_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230742171/Put/seqid=0 2024-11-10T09:25:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:25:42,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230802250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230802252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230802254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741959_1135 (size=14594) 2024-11-10T09:25:42,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230802256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230802258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:42,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:42,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230802366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230802366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230802366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230802376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230802376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:42,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:42,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:25:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230802586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230802586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230802587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230802587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230802587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:42,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107137a8ab98844c5aa075fe914ac563aa_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107137a8ab98844c5aa075fe914ac563aa_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:42,680 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/268d94727385427b8f5382e5260f47c8, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:42,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/268d94727385427b8f5382e5260f47c8 is 175, key is test_row_0/A:col10/1731230742171/Put/seqid=0 2024-11-10T09:25:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741960_1136 (size=39549) 2024-11-10T09:25:42,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/268d94727385427b8f5382e5260f47c8 2024-11-10T09:25:42,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/7cb8f43e5ed94ea48c30f34861412181 is 50, key is test_row_0/B:col10/1731230742171/Put/seqid=0 2024-11-10T09:25:42,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741961_1137 (size=12001) 2024-11-10T09:25:42,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:42,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:42,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:25:42,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230802896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230802896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230802896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:42,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:42,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:43,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:43,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:43,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/7cb8f43e5ed94ea48c30f34861412181 2024-11-10T09:25:43,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/9e226aafb535478a817e08a496e87e49 is 50, key is test_row_0/C:col10/1731230742171/Put/seqid=0 2024-11-10T09:25:43,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741962_1138 (size=12001) 2024-11-10T09:25:43,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/9e226aafb535478a817e08a496e87e49 2024-11-10T09:25:43,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/268d94727385427b8f5382e5260f47c8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/268d94727385427b8f5382e5260f47c8 2024-11-10T09:25:43,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/268d94727385427b8f5382e5260f47c8, entries=200, sequenceid=18, filesize=38.6 K 2024-11-10T09:25:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/7cb8f43e5ed94ea48c30f34861412181 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7cb8f43e5ed94ea48c30f34861412181 2024-11-10T09:25:43,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:43,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:43,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7cb8f43e5ed94ea48c30f34861412181, entries=150, sequenceid=18, filesize=11.7 K 2024-11-10T09:25:43,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:43,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/9e226aafb535478a817e08a496e87e49 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/9e226aafb535478a817e08a496e87e49 2024-11-10T09:25:43,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/9e226aafb535478a817e08a496e87e49, entries=150, sequenceid=18, filesize=11.7 K 2024-11-10T09:25:43,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1036ms, sequenceid=18, compaction requested=false 2024-11-10T09:25:43,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:25:43,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T09:25:43,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:43,386 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:25:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a5bb3af2fc0461a8e808448a19cbc1c_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230742250/Put/seqid=0 2024-11-10T09:25:43,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:43,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:43,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741963_1139 (size=12154) 2024-11-10T09:25:43,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:43,425 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106a5bb3af2fc0461a8e808448a19cbc1c_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a5bb3af2fc0461a8e808448a19cbc1c_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:43,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f15f6decd81a4e719adc27377dcdaf56, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:43,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f15f6decd81a4e719adc27377dcdaf56 is 175, key is test_row_0/A:col10/1731230742250/Put/seqid=0 2024-11-10T09:25:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230803425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230803425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230803425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230803425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230803428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741964_1140 (size=30955) 2024-11-10T09:25:43,440 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f15f6decd81a4e719adc27377dcdaf56 2024-11-10T09:25:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/7fc9fa869c934587aa52ece9cb48be77 is 50, key is test_row_0/B:col10/1731230742250/Put/seqid=0 2024-11-10T09:25:43,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741965_1141 (size=12001) 2024-11-10T09:25:43,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230803536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230803536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230803536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230803536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230803537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230803746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230803747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230803747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230803747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230803747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:43,826 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:25:43,856 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/7fc9fa869c934587aa52ece9cb48be77 2024-11-10T09:25:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/c190e43fe3c44d1abe253b7d5882f981 is 50, key is test_row_0/C:col10/1731230742250/Put/seqid=0 2024-11-10T09:25:43,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741966_1142 (size=12001) 2024-11-10T09:25:43,897 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/c190e43fe3c44d1abe253b7d5882f981 2024-11-10T09:25:43,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f15f6decd81a4e719adc27377dcdaf56 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f15f6decd81a4e719adc27377dcdaf56 2024-11-10T09:25:43,913 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f15f6decd81a4e719adc27377dcdaf56, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T09:25:43,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/7fc9fa869c934587aa52ece9cb48be77 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7fc9fa869c934587aa52ece9cb48be77 2024-11-10T09:25:43,928 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7fc9fa869c934587aa52ece9cb48be77, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T09:25:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/c190e43fe3c44d1abe253b7d5882f981 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/c190e43fe3c44d1abe253b7d5882f981 2024-11-10T09:25:43,949 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/c190e43fe3c44d1abe253b7d5882f981, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T09:25:43,951 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 565ms, sequenceid=40, compaction requested=false 2024-11-10T09:25:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T09:25:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T09:25:43,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T09:25:43,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-11-10T09:25:43,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8130 sec 2024-11-10T09:25:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:44,057 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:44,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T09:25:44,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111016d4cd2595c142c19093ed930897941a_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230744057/Put/seqid=0 2024-11-10T09:25:44,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741967_1143 (size=12154) 2024-11-10T09:25:44,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230804128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230804130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230804131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230804132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230804133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230804246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230804247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230804247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230804248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230804248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T09:25:44,296 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:44,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T09:25:44,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:25:44,301 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:44,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:25:44,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:25:44,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:44,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230804455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230804457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230804457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230804457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230804458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:44,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111016d4cd2595c142c19093ed930897941a_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111016d4cd2595c142c19093ed930897941a_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:44,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/29e3b85f23bf401886053f73a71e3dbf, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:44,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/29e3b85f23bf401886053f73a71e3dbf is 175, key is test_row_0/A:col10/1731230744057/Put/seqid=0 2024-11-10T09:25:44,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741968_1144 (size=30955) 2024-11-10T09:25:44,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/29e3b85f23bf401886053f73a71e3dbf 2024-11-10T09:25:44,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/046618aff0504b85bef816e9bdd53171 is 50, key is test_row_0/B:col10/1731230744057/Put/seqid=0 2024-11-10T09:25:44,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741969_1145 (size=12001) 2024-11-10T09:25:44,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:25:44,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:25:44,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:25:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230804767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230804767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230804767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230804767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230804767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:44,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:44,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:25:44,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:44,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:44,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/046618aff0504b85bef816e9bdd53171 2024-11-10T09:25:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:25:44,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/8bd898ca0a4a4f94b72941ea46921cad is 50, key is test_row_0/C:col10/1731230744057/Put/seqid=0 2024-11-10T09:25:44,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741970_1146 (size=12001) 2024-11-10T09:25:44,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/8bd898ca0a4a4f94b72941ea46921cad 2024-11-10T09:25:44,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/29e3b85f23bf401886053f73a71e3dbf as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/29e3b85f23bf401886053f73a71e3dbf 2024-11-10T09:25:44,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/29e3b85f23bf401886053f73a71e3dbf, entries=150, sequenceid=56, filesize=30.2 K 2024-11-10T09:25:44,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/046618aff0504b85bef816e9bdd53171 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/046618aff0504b85bef816e9bdd53171 2024-11-10T09:25:44,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/046618aff0504b85bef816e9bdd53171, entries=150, sequenceid=56, filesize=11.7 K 2024-11-10T09:25:44,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/8bd898ca0a4a4f94b72941ea46921cad as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8bd898ca0a4a4f94b72941ea46921cad 2024-11-10T09:25:44,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8bd898ca0a4a4f94b72941ea46921cad, entries=150, sequenceid=56, filesize=11.7 K 2024-11-10T09:25:44,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 936ms, sequenceid=56, compaction requested=true 2024-11-10T09:25:44,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:44,994 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:44,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:44,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:44,995 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:44,995 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/A is initiating minor compaction (all files) 2024-11-10T09:25:44,996 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/A in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,996 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/268d94727385427b8f5382e5260f47c8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f15f6decd81a4e719adc27377dcdaf56, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/29e3b85f23bf401886053f73a71e3dbf] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=99.1 K 2024-11-10T09:25:44,996 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,997 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/268d94727385427b8f5382e5260f47c8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f15f6decd81a4e719adc27377dcdaf56, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/29e3b85f23bf401886053f73a71e3dbf] 2024-11-10T09:25:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:44,997 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:44,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:44,998 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268d94727385427b8f5382e5260f47c8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731230742165 2024-11-10T09:25:44,998 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:44,998 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f15f6decd81a4e719adc27377dcdaf56, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230742250 2024-11-10T09:25:44,998 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/B is initiating minor compaction (all files) 2024-11-10T09:25:44,999 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/B in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:44,999 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7cb8f43e5ed94ea48c30f34861412181, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7fc9fa869c934587aa52ece9cb48be77, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/046618aff0504b85bef816e9bdd53171] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.2 K 2024-11-10T09:25:44,999 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e3b85f23bf401886053f73a71e3dbf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731230743416 2024-11-10T09:25:44,999 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cb8f43e5ed94ea48c30f34861412181, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731230742165 2024-11-10T09:25:45,000 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc9fa869c934587aa52ece9cb48be77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230742250 2024-11-10T09:25:45,000 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 046618aff0504b85bef816e9bdd53171, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731230743416 2024-11-10T09:25:45,007 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:45,011 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:45,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/f8806765fde94c268cdd3576f90bcba5 is 50, key is test_row_0/B:col10/1731230744057/Put/seqid=0 2024-11-10T09:25:45,024 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110770d11abdca84f0a9311cae70531d6d0_5619dfa0d7c1c633ebe4b8aecdc69e58 store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:45,034 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110770d11abdca84f0a9311cae70531d6d0_5619dfa0d7c1c633ebe4b8aecdc69e58, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:45,035 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110770d11abdca84f0a9311cae70531d6d0_5619dfa0d7c1c633ebe4b8aecdc69e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:45,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741971_1147 (size=12104) 2024-11-10T09:25:45,049 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/f8806765fde94c268cdd3576f90bcba5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f8806765fde94c268cdd3576f90bcba5 2024-11-10T09:25:45,059 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/B of 5619dfa0d7c1c633ebe4b8aecdc69e58 into f8806765fde94c268cdd3576f90bcba5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:45,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:45,060 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/B, priority=13, startTime=1731230744997; duration=0sec 2024-11-10T09:25:45,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:45,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:B 2024-11-10T09:25:45,060 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:45,061 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:45,062 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/C is initiating minor compaction (all files) 2024-11-10T09:25:45,062 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/C in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:45,062 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/9e226aafb535478a817e08a496e87e49, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/c190e43fe3c44d1abe253b7d5882f981, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8bd898ca0a4a4f94b72941ea46921cad] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.2 K 2024-11-10T09:25:45,062 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e226aafb535478a817e08a496e87e49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731230742165 2024-11-10T09:25:45,063 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c190e43fe3c44d1abe253b7d5882f981, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230742250 2024-11-10T09:25:45,065 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd898ca0a4a4f94b72941ea46921cad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731230743416 2024-11-10T09:25:45,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741972_1148 (size=4469) 2024-11-10T09:25:45,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T09:25:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:45,069 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T09:25:45,074 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#A#compaction#128 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:45,074 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d25b4006411e4521ad4a9226ea6e1104 is 175, key is test_row_0/A:col10/1731230744057/Put/seqid=0 2024-11-10T09:25:45,084 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#C#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:45,085 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/a6a50fed2ca04cc5b10bf190e6863c2a is 50, key is test_row_0/C:col10/1731230744057/Put/seqid=0 2024-11-10T09:25:45,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aee34dd0e5274dd8958319b82e191ca5_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230744131/Put/seqid=0 2024-11-10T09:25:45,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741973_1149 (size=31058) 2024-11-10T09:25:45,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741974_1150 (size=12104) 2024-11-10T09:25:45,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741975_1151 (size=12154) 2024-11-10T09:25:45,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,133 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110aee34dd0e5274dd8958319b82e191ca5_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110aee34dd0e5274dd8958319b82e191ca5_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:45,134 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/a6a50fed2ca04cc5b10bf190e6863c2a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/a6a50fed2ca04cc5b10bf190e6863c2a 2024-11-10T09:25:45,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/2f07b3d7dbfa40c39963d9d5bafab4e5, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:45,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/2f07b3d7dbfa40c39963d9d5bafab4e5 is 175, key is test_row_0/A:col10/1731230744131/Put/seqid=0 2024-11-10T09:25:45,144 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/C of 5619dfa0d7c1c633ebe4b8aecdc69e58 into a6a50fed2ca04cc5b10bf190e6863c2a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:45,144 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:45,144 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/C, priority=13, startTime=1731230744997; duration=0sec 2024-11-10T09:25:45,144 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:45,144 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:C 2024-11-10T09:25:45,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741976_1152 (size=30955) 2024-11-10T09:25:45,154 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/2f07b3d7dbfa40c39963d9d5bafab4e5 2024-11-10T09:25:45,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/36411f4a0e31457b9ddcf254e4b3e5e8 is 50, key is test_row_0/B:col10/1731230744131/Put/seqid=0 2024-11-10T09:25:45,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741977_1153 (size=12001) 2024-11-10T09:25:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:45,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:45,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230805298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230805298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230805300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230805301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230805303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230805416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230805416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230805416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230805416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230805416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:25:45,512 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d25b4006411e4521ad4a9226ea6e1104 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d25b4006411e4521ad4a9226ea6e1104 2024-11-10T09:25:45,540 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/A of 5619dfa0d7c1c633ebe4b8aecdc69e58 into d25b4006411e4521ad4a9226ea6e1104(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:45,540 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:45,540 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/A, priority=13, startTime=1731230744994; duration=0sec 2024-11-10T09:25:45,540 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:45,540 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:A 2024-11-10T09:25:45,591 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/36411f4a0e31457b9ddcf254e4b3e5e8 2024-11-10T09:25:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/1a838e27445b4ca482aed40bb6821224 is 50, key is test_row_0/C:col10/1731230744131/Put/seqid=0 2024-11-10T09:25:45,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741978_1154 (size=12001) 2024-11-10T09:25:45,610 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/1a838e27445b4ca482aed40bb6821224 2024-11-10T09:25:45,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/2f07b3d7dbfa40c39963d9d5bafab4e5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/2f07b3d7dbfa40c39963d9d5bafab4e5 2024-11-10T09:25:45,622 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/2f07b3d7dbfa40c39963d9d5bafab4e5, entries=150, sequenceid=78, filesize=30.2 K 2024-11-10T09:25:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/36411f4a0e31457b9ddcf254e4b3e5e8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/36411f4a0e31457b9ddcf254e4b3e5e8 2024-11-10T09:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230805626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230805626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230805626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230805626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230805626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:45,631 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/36411f4a0e31457b9ddcf254e4b3e5e8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T09:25:45,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/1a838e27445b4ca482aed40bb6821224 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/1a838e27445b4ca482aed40bb6821224 2024-11-10T09:25:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,638 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/1a838e27445b4ca482aed40bb6821224, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T09:25:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,639 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 570ms, sequenceid=78, compaction requested=false 2024-11-10T09:25:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T09:25:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T09:25:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T09:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3400 sec 2024-11-10T09:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3480 sec 2024-11-10T09:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:45,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T09:25:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100acec475dfb84181a97c812e1bd3b942_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230745300/Put/seqid=0 2024-11-10T09:25:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741979_1155 (size=12154) 2024-11-10T09:25:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:45,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100acec475dfb84181a97c812e1bd3b942_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100acec475dfb84181a97c812e1bd3b942_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:45,986 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/7f47ec181864479e8a93c28b38db2be4, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:45,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/7f47ec181864479e8a93c28b38db2be4 is 175, key is test_row_0/A:col10/1731230745300/Put/seqid=0 2024-11-10T09:25:46,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741980_1156 (size=30955) 2024-11-10T09:25:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230806019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230806020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230806021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230806021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230806021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230806136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230806136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230806136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230806136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230806136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230806346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230806346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230806346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230806346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230806346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,408 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/7f47ec181864479e8a93c28b38db2be4 2024-11-10T09:25:46,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/1ea1efca0036482c9de97f21875c4c0b is 50, key is test_row_0/B:col10/1731230745300/Put/seqid=0 2024-11-10T09:25:46,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741981_1157 (size=12001) 2024-11-10T09:25:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T09:25:46,445 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:46,446 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T09:25:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:25:46,448 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:46,448 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:46,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:25:46,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:25:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:46,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230806655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230806656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230806656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230806666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:46,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230806666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:46,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:46,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:46,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:25:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:25:46,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/1ea1efca0036482c9de97f21875c4c0b 2024-11-10T09:25:46,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/10417dbcfc69403bb55dba9f9bf74a64 is 50, key is test_row_0/C:col10/1731230745300/Put/seqid=0 2024-11-10T09:25:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741982_1158 (size=12001) 2024-11-10T09:25:46,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/10417dbcfc69403bb55dba9f9bf74a64 2024-11-10T09:25:46,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/7f47ec181864479e8a93c28b38db2be4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/7f47ec181864479e8a93c28b38db2be4 2024-11-10T09:25:46,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/7f47ec181864479e8a93c28b38db2be4, entries=150, sequenceid=96, filesize=30.2 K 2024-11-10T09:25:46,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/1ea1efca0036482c9de97f21875c4c0b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/1ea1efca0036482c9de97f21875c4c0b 2024-11-10T09:25:46,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/1ea1efca0036482c9de97f21875c4c0b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T09:25:46,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/10417dbcfc69403bb55dba9f9bf74a64 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/10417dbcfc69403bb55dba9f9bf74a64 2024-11-10T09:25:46,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/10417dbcfc69403bb55dba9f9bf74a64, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T09:25:46,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 912ms, sequenceid=96, compaction requested=true 2024-11-10T09:25:46,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:46,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:46,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:46,861 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:46,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:46,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:46,861 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:46,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:46,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:46,862 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:46,862 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:46,862 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/B is initiating minor compaction (all files) 2024-11-10T09:25:46,862 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/A is initiating minor compaction (all files) 2024-11-10T09:25:46,862 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/A in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,862 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/B in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,862 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f8806765fde94c268cdd3576f90bcba5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/36411f4a0e31457b9ddcf254e4b3e5e8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/1ea1efca0036482c9de97f21875c4c0b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.3 K 2024-11-10T09:25:46,862 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d25b4006411e4521ad4a9226ea6e1104, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/2f07b3d7dbfa40c39963d9d5bafab4e5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/7f47ec181864479e8a93c28b38db2be4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=90.8 K 2024-11-10T09:25:46,862 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,862 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d25b4006411e4521ad4a9226ea6e1104, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/2f07b3d7dbfa40c39963d9d5bafab4e5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/7f47ec181864479e8a93c28b38db2be4] 2024-11-10T09:25:46,863 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f8806765fde94c268cdd3576f90bcba5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731230743416 2024-11-10T09:25:46,863 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d25b4006411e4521ad4a9226ea6e1104, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731230743416 2024-11-10T09:25:46,863 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 36411f4a0e31457b9ddcf254e4b3e5e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731230744120 2024-11-10T09:25:46,863 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f07b3d7dbfa40c39963d9d5bafab4e5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731230744120 2024-11-10T09:25:46,865 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea1efca0036482c9de97f21875c4c0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731230745291 2024-11-10T09:25:46,865 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f47ec181864479e8a93c28b38db2be4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731230745291 2024-11-10T09:25:46,875 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:46,882 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:46,884 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/925e0218f6f54372996012cd6d3cfd05 is 50, key is test_row_0/B:col10/1731230745300/Put/seqid=0 2024-11-10T09:25:46,897 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110e12fd57d193b477ea8e3c491ce5e28ba_5619dfa0d7c1c633ebe4b8aecdc69e58 store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:46,900 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110e12fd57d193b477ea8e3c491ce5e28ba_5619dfa0d7c1c633ebe4b8aecdc69e58, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:46,900 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e12fd57d193b477ea8e3c491ce5e28ba_5619dfa0d7c1c633ebe4b8aecdc69e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:46,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T09:25:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:46,911 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T09:25:46,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741984_1160 (size=4469) 2024-11-10T09:25:46,916 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#A#compaction#137 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:46,917 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/e81c1215822a4affbee4c710ac3ff440 is 175, key is test_row_0/A:col10/1731230745300/Put/seqid=0 2024-11-10T09:25:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741983_1159 (size=12207) 2024-11-10T09:25:46,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110927820a3dee64a15a9e6be02d9c960d8_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230746017/Put/seqid=0 2024-11-10T09:25:46,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741985_1161 (size=31161) 2024-11-10T09:25:46,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741986_1162 (size=12154) 2024-11-10T09:25:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:25:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:47,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:47,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230807183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230807183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230807188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230807189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230807191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230807296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230807296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230807305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230807305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230807305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,328 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/925e0218f6f54372996012cd6d3cfd05 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/925e0218f6f54372996012cd6d3cfd05 2024-11-10T09:25:47,333 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/B of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 925e0218f6f54372996012cd6d3cfd05(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:47,333 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:47,335 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/B, priority=13, startTime=1731230746861; duration=0sec 2024-11-10T09:25:47,336 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:47,336 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:B 2024-11-10T09:25:47,336 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:47,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:47,344 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:47,344 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/C is initiating minor compaction (all files) 2024-11-10T09:25:47,344 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/C in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:47,344 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/a6a50fed2ca04cc5b10bf190e6863c2a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/1a838e27445b4ca482aed40bb6821224, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/10417dbcfc69403bb55dba9f9bf74a64] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.3 K 2024-11-10T09:25:47,345 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a6a50fed2ca04cc5b10bf190e6863c2a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731230743416 2024-11-10T09:25:47,345 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a838e27445b4ca482aed40bb6821224, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731230744120 2024-11-10T09:25:47,345 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/e81c1215822a4affbee4c710ac3ff440 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e81c1215822a4affbee4c710ac3ff440 2024-11-10T09:25:47,345 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 10417dbcfc69403bb55dba9f9bf74a64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731230745291 2024-11-10T09:25:47,347 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110927820a3dee64a15a9e6be02d9c960d8_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110927820a3dee64a15a9e6be02d9c960d8_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:47,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/e1c5cec4652b40e289410cf30cab4311, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:47,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/e1c5cec4652b40e289410cf30cab4311 is 175, key is test_row_0/A:col10/1731230746017/Put/seqid=0 2024-11-10T09:25:47,352 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/A of 5619dfa0d7c1c633ebe4b8aecdc69e58 into e81c1215822a4affbee4c710ac3ff440(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:47,353 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:47,353 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/A, priority=13, startTime=1731230746861; duration=0sec 2024-11-10T09:25:47,353 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:47,353 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:A 2024-11-10T09:25:47,359 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#C#compaction#140 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:47,360 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/e23d00d1b6b74f8cae4a23e79834468b is 50, key is test_row_0/C:col10/1731230745300/Put/seqid=0 2024-11-10T09:25:47,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741987_1163 (size=30955) 2024-11-10T09:25:47,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741988_1164 (size=12207) 2024-11-10T09:25:47,373 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/e1c5cec4652b40e289410cf30cab4311 2024-11-10T09:25:47,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/db143fbf55ac43358550ef57f34c6618 is 50, key is test_row_0/B:col10/1731230746017/Put/seqid=0 2024-11-10T09:25:47,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741989_1165 (size=12001) 2024-11-10T09:25:47,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230807505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230807505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230807515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230807515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230807516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:25:47,779 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/e23d00d1b6b74f8cae4a23e79834468b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/e23d00d1b6b74f8cae4a23e79834468b 2024-11-10T09:25:47,785 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/C of 5619dfa0d7c1c633ebe4b8aecdc69e58 into e23d00d1b6b74f8cae4a23e79834468b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:47,785 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:47,785 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/C, priority=13, startTime=1731230746861; duration=0sec 2024-11-10T09:25:47,785 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:47,785 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/db143fbf55ac43358550ef57f34c6618 2024-11-10T09:25:47,785 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:C 2024-11-10T09:25:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/fb06a5d2fa554486a53d913ef37a653d is 50, key is test_row_0/C:col10/1731230746017/Put/seqid=0 2024-11-10T09:25:47,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741990_1166 (size=12001) 2024-11-10T09:25:47,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230807815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230807815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230807826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230807826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230807826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,207 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/fb06a5d2fa554486a53d913ef37a653d 2024-11-10T09:25:48,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/e1c5cec4652b40e289410cf30cab4311 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e1c5cec4652b40e289410cf30cab4311 2024-11-10T09:25:48,216 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e1c5cec4652b40e289410cf30cab4311, entries=150, sequenceid=117, filesize=30.2 K 2024-11-10T09:25:48,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/db143fbf55ac43358550ef57f34c6618 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/db143fbf55ac43358550ef57f34c6618 2024-11-10T09:25:48,221 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/db143fbf55ac43358550ef57f34c6618, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:25:48,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/fb06a5d2fa554486a53d913ef37a653d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/fb06a5d2fa554486a53d913ef37a653d 2024-11-10T09:25:48,226 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/fb06a5d2fa554486a53d913ef37a653d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T09:25:48,227 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1316ms, sequenceid=117, compaction requested=false 2024-11-10T09:25:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T09:25:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T09:25:48,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T09:25:48,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-11-10T09:25:48,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.7850 sec 2024-11-10T09:25:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:48,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T09:25:48,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108115e58caa024bc08eb378d1e69cae80_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230748326/Put/seqid=0 2024-11-10T09:25:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230808347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230808347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230808349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230808351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230808353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741991_1167 (size=17234) 2024-11-10T09:25:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230808456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230808456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230808456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230808465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230808465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T09:25:48,596 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:48,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T09:25:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:25:48,600 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:48,601 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:48,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230808665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230808665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230808665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230808676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230808676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:25:48,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:25:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:48,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:48,765 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:48,769 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108115e58caa024bc08eb378d1e69cae80_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108115e58caa024bc08eb378d1e69cae80_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:48,771 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d77a91ef82964e45b37b42e6af549ccd, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:48,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d77a91ef82964e45b37b42e6af549ccd is 175, key is test_row_0/A:col10/1731230748326/Put/seqid=0 2024-11-10T09:25:48,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741992_1168 (size=48339) 2024-11-10T09:25:48,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:25:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:48,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:48,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:25:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230808975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230808975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230808975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230808986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230808986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:49,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:25:49,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:49,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d77a91ef82964e45b37b42e6af549ccd 2024-11-10T09:25:49,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/77949c0d05fb4bd2a2c5293a8f75ca9f is 50, key is test_row_0/B:col10/1731230748326/Put/seqid=0 2024-11-10T09:25:49,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741993_1169 (size=12101) 2024-11-10T09:25:49,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/77949c0d05fb4bd2a2c5293a8f75ca9f 2024-11-10T09:25:49,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/29385bbc400448de8994b4c7e914a5e7 is 50, key is test_row_0/C:col10/1731230748326/Put/seqid=0 2024-11-10T09:25:49,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741994_1170 (size=12101) 2024-11-10T09:25:49,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:25:49,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:49,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:25:49,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:25:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230809486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230809486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230809486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:49,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230809495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:49,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:49,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230809495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:49,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:49,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:49,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:49,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:25:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:49,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/29385bbc400448de8994b4c7e914a5e7 2024-11-10T09:25:49,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d77a91ef82964e45b37b42e6af549ccd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d77a91ef82964e45b37b42e6af549ccd 2024-11-10T09:25:49,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d77a91ef82964e45b37b42e6af549ccd, entries=250, sequenceid=137, filesize=47.2 K 2024-11-10T09:25:49,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/77949c0d05fb4bd2a2c5293a8f75ca9f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/77949c0d05fb4bd2a2c5293a8f75ca9f 2024-11-10T09:25:49,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/77949c0d05fb4bd2a2c5293a8f75ca9f, entries=150, sequenceid=137, filesize=11.8 K 2024-11-10T09:25:49,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/29385bbc400448de8994b4c7e914a5e7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/29385bbc400448de8994b4c7e914a5e7 2024-11-10T09:25:49,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/29385bbc400448de8994b4c7e914a5e7, entries=150, sequenceid=137, filesize=11.8 K 2024-11-10T09:25:49,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1310ms, sequenceid=137, compaction requested=true 2024-11-10T09:25:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:49,638 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:49,638 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:49,639 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:49,640 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/B is initiating minor compaction (all files) 2024-11-10T09:25:49,640 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/B in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,640 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/925e0218f6f54372996012cd6d3cfd05, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/db143fbf55ac43358550ef57f34c6618, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/77949c0d05fb4bd2a2c5293a8f75ca9f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.5 K 2024-11-10T09:25:49,640 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:49,640 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/A is initiating minor compaction (all files) 2024-11-10T09:25:49,640 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/A in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,640 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 925e0218f6f54372996012cd6d3cfd05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731230745291 2024-11-10T09:25:49,640 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e81c1215822a4affbee4c710ac3ff440, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e1c5cec4652b40e289410cf30cab4311, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d77a91ef82964e45b37b42e6af549ccd] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=107.9 K 2024-11-10T09:25:49,640 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,640 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e81c1215822a4affbee4c710ac3ff440, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e1c5cec4652b40e289410cf30cab4311, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d77a91ef82964e45b37b42e6af549ccd] 2024-11-10T09:25:49,641 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting db143fbf55ac43358550ef57f34c6618, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230746006 2024-11-10T09:25:49,641 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e81c1215822a4affbee4c710ac3ff440, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731230745291 2024-11-10T09:25:49,641 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 77949c0d05fb4bd2a2c5293a8f75ca9f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731230747181 2024-11-10T09:25:49,641 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1c5cec4652b40e289410cf30cab4311, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230746006 2024-11-10T09:25:49,642 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d77a91ef82964e45b37b42e6af549ccd, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731230747181 2024-11-10T09:25:49,657 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#B#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:49,658 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/bab9a2997423438d8fa90bb08037be16 is 50, key is test_row_0/B:col10/1731230748326/Put/seqid=0 2024-11-10T09:25:49,660 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:49,669 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411101198d9bd0efb421387361bba090828e1_5619dfa0d7c1c633ebe4b8aecdc69e58 store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:49,671 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411101198d9bd0efb421387361bba090828e1_5619dfa0d7c1c633ebe4b8aecdc69e58, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:49,671 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101198d9bd0efb421387361bba090828e1_5619dfa0d7c1c633ebe4b8aecdc69e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:49,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T09:25:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,672 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T09:25:49,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741995_1171 (size=12409) 2024-11-10T09:25:49,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c238fbf5e3d456ca640ed5ca5220887_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230748340/Put/seqid=0 2024-11-10T09:25:49,702 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/bab9a2997423438d8fa90bb08037be16 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/bab9a2997423438d8fa90bb08037be16 2024-11-10T09:25:49,709 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/B of 5619dfa0d7c1c633ebe4b8aecdc69e58 into bab9a2997423438d8fa90bb08037be16(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:49,709 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:49,709 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/B, priority=13, startTime=1731230749638; duration=0sec 2024-11-10T09:25:49,710 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:49,710 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:B 2024-11-10T09:25:49,710 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741996_1172 (size=4469) 2024-11-10T09:25:49,711 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:49,711 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/C is initiating minor compaction (all files) 2024-11-10T09:25:49,711 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/C in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:49,711 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/e23d00d1b6b74f8cae4a23e79834468b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/fb06a5d2fa554486a53d913ef37a653d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/29385bbc400448de8994b4c7e914a5e7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.5 K 2024-11-10T09:25:49,711 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e23d00d1b6b74f8cae4a23e79834468b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731230745291 2024-11-10T09:25:49,712 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting fb06a5d2fa554486a53d913ef37a653d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731230746006 2024-11-10T09:25:49,712 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 29385bbc400448de8994b4c7e914a5e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731230747181 2024-11-10T09:25:49,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741997_1173 (size=12304) 2024-11-10T09:25:49,719 INFO [master/802346a334f5:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T09:25:49,719 INFO [master/802346a334f5:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T09:25:49,724 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:49,724 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/7f5a47555a2a40eeb1ab40167264c5b6 is 50, key is test_row_0/C:col10/1731230748326/Put/seqid=0 2024-11-10T09:25:49,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741998_1174 (size=12409) 2024-11-10T09:25:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:25:49,743 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/7f5a47555a2a40eeb1ab40167264c5b6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7f5a47555a2a40eeb1ab40167264c5b6 2024-11-10T09:25:49,752 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/C of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 7f5a47555a2a40eeb1ab40167264c5b6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:49,752 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:49,752 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/C, priority=13, startTime=1731230749638; duration=0sec 2024-11-10T09:25:49,752 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:49,752 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:C 2024-11-10T09:25:50,111 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:50,111 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/3cb18c50d1ef47958ebac827db4abbeb is 175, key is test_row_0/A:col10/1731230748326/Put/seqid=0 2024-11-10T09:25:50,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741999_1175 (size=31363) 2024-11-10T09:25:50,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:50,121 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c238fbf5e3d456ca640ed5ca5220887_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c238fbf5e3d456ca640ed5ca5220887_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/02dbf54792c441af91ae53082abcd69c, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:50,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/02dbf54792c441af91ae53082abcd69c is 175, key is test_row_0/A:col10/1731230748340/Put/seqid=0 2024-11-10T09:25:50,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742000_1176 (size=31105) 2024-11-10T09:25:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:50,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:50,520 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/3cb18c50d1ef47958ebac827db4abbeb as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/3cb18c50d1ef47958ebac827db4abbeb 2024-11-10T09:25:50,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230810513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,525 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/A of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 3cb18c50d1ef47958ebac827db4abbeb(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:50,525 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:50,525 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/A, priority=13, startTime=1731230749638; duration=0sec 2024-11-10T09:25:50,525 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:50,525 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:A 2024-11-10T09:25:50,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230810514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230810520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230810523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230810521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,528 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/02dbf54792c441af91ae53082abcd69c 2024-11-10T09:25:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/85fe951ff7e94762b57fbc5040fd019c is 50, key is test_row_0/B:col10/1731230748340/Put/seqid=0 2024-11-10T09:25:50,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742001_1177 (size=12151) 2024-11-10T09:25:50,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230810625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230810636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230810636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230810636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230810637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:25:50,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230810836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230810846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230810846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230810846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230810846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:50,960 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/85fe951ff7e94762b57fbc5040fd019c 2024-11-10T09:25:50,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/d890755d1e6440258a5cec42e5e45295 is 50, key is test_row_0/C:col10/1731230748340/Put/seqid=0 2024-11-10T09:25:50,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742002_1178 (size=12151) 2024-11-10T09:25:51,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230811146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230811156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230811156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230811156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230811156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,374 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/d890755d1e6440258a5cec42e5e45295 2024-11-10T09:25:51,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/02dbf54792c441af91ae53082abcd69c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/02dbf54792c441af91ae53082abcd69c 2024-11-10T09:25:51,386 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/02dbf54792c441af91ae53082abcd69c, entries=150, sequenceid=156, filesize=30.4 K 2024-11-10T09:25:51,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/85fe951ff7e94762b57fbc5040fd019c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/85fe951ff7e94762b57fbc5040fd019c 2024-11-10T09:25:51,392 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/85fe951ff7e94762b57fbc5040fd019c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T09:25:51,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/d890755d1e6440258a5cec42e5e45295 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/d890755d1e6440258a5cec42e5e45295 2024-11-10T09:25:51,397 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/d890755d1e6440258a5cec42e5e45295, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T09:25:51,397 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1725ms, sequenceid=156, compaction requested=false 2024-11-10T09:25:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:51,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T09:25:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T09:25:51,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T09:25:51,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7980 sec 2024-11-10T09:25:51,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.8050 sec 2024-11-10T09:25:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:51,657 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:51,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T09:25:51,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111010df65902baf444cb588fd25cb902c92_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230751655/Put/seqid=0 2024-11-10T09:25:51,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230811683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230811684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230811685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230811687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742003_1179 (size=14794) 2024-11-10T09:25:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230811688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230811795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230811795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230811795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230811795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:51,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230811806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:51,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230812005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230812005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230812005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230812006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230812016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,093 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:52,097 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111010df65902baf444cb588fd25cb902c92_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111010df65902baf444cb588fd25cb902c92_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:52,098 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/daf955e5318d48e68f12e8914f039a9c, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:52,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/daf955e5318d48e68f12e8914f039a9c is 175, key is test_row_0/A:col10/1731230751655/Put/seqid=0 2024-11-10T09:25:52,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742004_1180 (size=39749) 2024-11-10T09:25:52,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230812315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230812315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230812315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230812316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230812326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,502 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/daf955e5318d48e68f12e8914f039a9c 2024-11-10T09:25:52,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/265e7c0fe30446feb132e1951af9ef7e is 50, key is test_row_0/B:col10/1731230751655/Put/seqid=0 2024-11-10T09:25:52,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742005_1181 (size=12151) 2024-11-10T09:25:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T09:25:52,765 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:52,766 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T09:25:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:25:52,767 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:52,768 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:52,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:52,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230812826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230812826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230812826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230812826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230812835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:25:52,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T09:25:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:52,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:52,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/265e7c0fe30446feb132e1951af9ef7e 2024-11-10T09:25:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:52,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/8e02cb8cee1a4fb39cd33667b00bbdb1 is 50, key is test_row_0/C:col10/1731230751655/Put/seqid=0 2024-11-10T09:25:52,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742006_1182 (size=12151) 2024-11-10T09:25:53,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T09:25:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:25:53,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T09:25:53,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:53,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/8e02cb8cee1a4fb39cd33667b00bbdb1 2024-11-10T09:25:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/daf955e5318d48e68f12e8914f039a9c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/daf955e5318d48e68f12e8914f039a9c 2024-11-10T09:25:53,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/daf955e5318d48e68f12e8914f039a9c, entries=200, sequenceid=177, filesize=38.8 K 2024-11-10T09:25:53,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/265e7c0fe30446feb132e1951af9ef7e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/265e7c0fe30446feb132e1951af9ef7e 2024-11-10T09:25:53,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/265e7c0fe30446feb132e1951af9ef7e, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T09:25:53,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/8e02cb8cee1a4fb39cd33667b00bbdb1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8e02cb8cee1a4fb39cd33667b00bbdb1 2024-11-10T09:25:53,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8e02cb8cee1a4fb39cd33667b00bbdb1, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T09:25:53,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1693ms, sequenceid=177, compaction requested=true 2024-11-10T09:25:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:53,351 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:53,351 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:53,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:53,352 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:53,352 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/B is initiating minor compaction (all files) 2024-11-10T09:25:53,352 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/B in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,352 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/bab9a2997423438d8fa90bb08037be16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/85fe951ff7e94762b57fbc5040fd019c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/265e7c0fe30446feb132e1951af9ef7e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.9 K 2024-11-10T09:25:53,352 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:53,352 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/A is initiating minor compaction (all files) 2024-11-10T09:25:53,352 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/A in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,353 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/3cb18c50d1ef47958ebac827db4abbeb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/02dbf54792c441af91ae53082abcd69c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/daf955e5318d48e68f12e8914f039a9c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=99.8 K 2024-11-10T09:25:53,353 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,353 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/3cb18c50d1ef47958ebac827db4abbeb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/02dbf54792c441af91ae53082abcd69c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/daf955e5318d48e68f12e8914f039a9c] 2024-11-10T09:25:53,353 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting bab9a2997423438d8fa90bb08037be16, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731230747181 2024-11-10T09:25:53,353 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cb18c50d1ef47958ebac827db4abbeb, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731230747181 2024-11-10T09:25:53,353 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 85fe951ff7e94762b57fbc5040fd019c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731230748340 2024-11-10T09:25:53,354 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02dbf54792c441af91ae53082abcd69c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731230748340 2024-11-10T09:25:53,354 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 265e7c0fe30446feb132e1951af9ef7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731230750520 2024-11-10T09:25:53,354 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf955e5318d48e68f12e8914f039a9c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731230750510 2024-11-10T09:25:53,361 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:53,362 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:53,363 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/ca2ec49a661e46e097c4e048d3be5ff1 is 50, key is test_row_0/B:col10/1731230751655/Put/seqid=0 2024-11-10T09:25:53,365 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111060f6f6259b9142af9dadd0bd149b3a34_5619dfa0d7c1c633ebe4b8aecdc69e58 store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:53,368 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111060f6f6259b9142af9dadd0bd149b3a34_5619dfa0d7c1c633ebe4b8aecdc69e58, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:53,368 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111060f6f6259b9142af9dadd0bd149b3a34_5619dfa0d7c1c633ebe4b8aecdc69e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:53,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742007_1183 (size=12561) 2024-11-10T09:25:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742008_1184 (size=4469) 2024-11-10T09:25:53,376 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#A#compaction#155 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:53,376 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/a09dd52ec38f407dac5102174727338f is 175, key is test_row_0/A:col10/1731230751655/Put/seqid=0 2024-11-10T09:25:53,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T09:25:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,379 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-10T09:25:53,381 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/ca2ec49a661e46e097c4e048d3be5ff1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/ca2ec49a661e46e097c4e048d3be5ff1 2024-11-10T09:25:53,390 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/B of 5619dfa0d7c1c633ebe4b8aecdc69e58 into ca2ec49a661e46e097c4e048d3be5ff1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:53,390 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:53,390 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/B, priority=13, startTime=1731230753351; duration=0sec 2024-11-10T09:25:53,390 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:53,390 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:B 2024-11-10T09:25:53,391 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:53,393 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:53,393 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/C is initiating minor compaction (all files) 2024-11-10T09:25:53,393 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/C in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:53,393 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7f5a47555a2a40eeb1ab40167264c5b6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/d890755d1e6440258a5cec42e5e45295, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8e02cb8cee1a4fb39cd33667b00bbdb1] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=35.9 K 2024-11-10T09:25:53,393 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f5a47555a2a40eeb1ab40167264c5b6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731230747181 2024-11-10T09:25:53,394 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d890755d1e6440258a5cec42e5e45295, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731230748340 2024-11-10T09:25:53,394 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e02cb8cee1a4fb39cd33667b00bbdb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731230750520 2024-11-10T09:25:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:25:53,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742009_1185 (size=31515) 2024-11-10T09:25:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c9f30a23cdab4a0b9e774544fcc3a8c3_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230751682/Put/seqid=0 2024-11-10T09:25:53,410 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#C#compaction#158 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:53,412 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/41eb7c424bce44e188b3a6dd09ad1352 is 50, key is test_row_0/C:col10/1731230751655/Put/seqid=0 2024-11-10T09:25:53,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742010_1186 (size=12304) 2024-11-10T09:25:53,414 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/a09dd52ec38f407dac5102174727338f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a09dd52ec38f407dac5102174727338f 2024-11-10T09:25:53,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:53,420 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c9f30a23cdab4a0b9e774544fcc3a8c3_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c9f30a23cdab4a0b9e774544fcc3a8c3_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/64d8a7d2c4804a60af63bc97e7454dee, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:53,421 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/A of 5619dfa0d7c1c633ebe4b8aecdc69e58 into a09dd52ec38f407dac5102174727338f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:53,421 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:53,421 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/A, priority=13, startTime=1731230753351; duration=0sec 2024-11-10T09:25:53,422 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:53,422 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:A 2024-11-10T09:25:53,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/64d8a7d2c4804a60af63bc97e7454dee is 175, key is test_row_0/A:col10/1731230751682/Put/seqid=0 2024-11-10T09:25:53,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742011_1187 (size=12561) 2024-11-10T09:25:53,434 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/41eb7c424bce44e188b3a6dd09ad1352 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/41eb7c424bce44e188b3a6dd09ad1352 2024-11-10T09:25:53,446 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/C of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 41eb7c424bce44e188b3a6dd09ad1352(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:53,447 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:53,447 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/C, priority=13, startTime=1731230753351; duration=0sec 2024-11-10T09:25:53,447 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:53,447 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:C 2024-11-10T09:25:53,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742012_1188 (size=31105) 2024-11-10T09:25:53,458 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/64d8a7d2c4804a60af63bc97e7454dee 2024-11-10T09:25:53,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/46018b55b8ff4bb892f193dca3d22ae3 is 50, key is test_row_0/B:col10/1731230751682/Put/seqid=0 2024-11-10T09:25:53,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742013_1189 (size=12151) 2024-11-10T09:25:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:53,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:53,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230813851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230813851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230813852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230813853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230813856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,861 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,871 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/46018b55b8ff4bb892f193dca3d22ae3 2024-11-10T09:25:53,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/3d5eee65e532436f959af8d513e478de is 50, key is test_row_0/C:col10/1731230751682/Put/seqid=0 2024-11-10T09:25:53,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742014_1190 (size=12151) 2024-11-10T09:25:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:25:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230813956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230813965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230813965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230813965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230814166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230814175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230814175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230814175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,295 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/3d5eee65e532436f959af8d513e478de 2024-11-10T09:25:54,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/64d8a7d2c4804a60af63bc97e7454dee as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/64d8a7d2c4804a60af63bc97e7454dee 2024-11-10T09:25:54,303 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/64d8a7d2c4804a60af63bc97e7454dee, entries=150, sequenceid=195, filesize=30.4 K 2024-11-10T09:25:54,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/46018b55b8ff4bb892f193dca3d22ae3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/46018b55b8ff4bb892f193dca3d22ae3 2024-11-10T09:25:54,308 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/46018b55b8ff4bb892f193dca3d22ae3, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:25:54,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/3d5eee65e532436f959af8d513e478de as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/3d5eee65e532436f959af8d513e478de 2024-11-10T09:25:54,315 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/3d5eee65e532436f959af8d513e478de, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:25:54,316 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 936ms, sequenceid=195, compaction requested=false 2024-11-10T09:25:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:54,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T09:25:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T09:25:54,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T09:25:54,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5490 sec 2024-11-10T09:25:54,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5540 sec 2024-11-10T09:25:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:54,478 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:54,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-10T09:25:54,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d31a364300cc4bdd8435a699de7b8df9_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230753848/Put/seqid=0 2024-11-10T09:25:54,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230814499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230814499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230814503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230814504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742015_1191 (size=14794) 2024-11-10T09:25:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230814615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230814615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230814615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230814615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230814825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230814825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230814825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230814825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:54,911 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:54,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d31a364300cc4bdd8435a699de7b8df9_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d31a364300cc4bdd8435a699de7b8df9_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:54,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d5a207a634eb4487af578be1e9c2564a, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:54,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d5a207a634eb4487af578be1e9c2564a is 175, key is test_row_0/A:col10/1731230753848/Put/seqid=0 2024-11-10T09:25:54,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742016_1192 (size=39749) 2024-11-10T09:25:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T09:25:54,925 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:54,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T09:25:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:25:54,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:54,928 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:54,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:25:55,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:25:55,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:55,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:25:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:25:55,321 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d5a207a634eb4487af578be1e9c2564a 2024-11-10T09:25:55,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/5b598b35706b4d1ab3df62062355ca58 is 50, key is test_row_0/B:col10/1731230753848/Put/seqid=0 2024-11-10T09:25:55,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742017_1193 (size=12151) 2024-11-10T09:25:55,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/5b598b35706b4d1ab3df62062355ca58 2024-11-10T09:25:55,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/08bfc8f71a7e423ab4d202620fd11de0 is 50, key is test_row_0/C:col10/1731230753848/Put/seqid=0 2024-11-10T09:25:55,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742018_1194 (size=12151) 2024-11-10T09:25:55,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:25:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:25:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:25:55,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230815645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230815655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230815655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230815655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:55,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:25:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:55,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:55,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/08bfc8f71a7e423ab4d202620fd11de0 2024-11-10T09:25:55,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/d5a207a634eb4487af578be1e9c2564a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d5a207a634eb4487af578be1e9c2564a 2024-11-10T09:25:55,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d5a207a634eb4487af578be1e9c2564a, entries=200, sequenceid=217, filesize=38.8 K 2024-11-10T09:25:55,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/5b598b35706b4d1ab3df62062355ca58 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/5b598b35706b4d1ab3df62062355ca58 2024-11-10T09:25:55,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/5b598b35706b4d1ab3df62062355ca58, entries=150, sequenceid=217, filesize=11.9 K 2024-11-10T09:25:55,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/08bfc8f71a7e423ab4d202620fd11de0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/08bfc8f71a7e423ab4d202620fd11de0 2024-11-10T09:25:55,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/08bfc8f71a7e423ab4d202620fd11de0, entries=150, sequenceid=217, filesize=11.9 K 2024-11-10T09:25:55,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1289ms, sequenceid=217, compaction requested=true 2024-11-10T09:25:55,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:55,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:55,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:55,767 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:55,768 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:55,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:55,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:55,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:55,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:55,768 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:55,768 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:55,768 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/B is initiating minor compaction (all files) 2024-11-10T09:25:55,769 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/A is initiating minor compaction (all files) 2024-11-10T09:25:55,769 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/B in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,769 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/A in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,769 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/ca2ec49a661e46e097c4e048d3be5ff1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/46018b55b8ff4bb892f193dca3d22ae3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/5b598b35706b4d1ab3df62062355ca58] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=36.0 K 2024-11-10T09:25:55,769 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a09dd52ec38f407dac5102174727338f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/64d8a7d2c4804a60af63bc97e7454dee, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d5a207a634eb4487af578be1e9c2564a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=100.0 K 2024-11-10T09:25:55,769 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,769 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a09dd52ec38f407dac5102174727338f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/64d8a7d2c4804a60af63bc97e7454dee, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d5a207a634eb4487af578be1e9c2564a] 2024-11-10T09:25:55,769 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2ec49a661e46e097c4e048d3be5ff1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731230750520 2024-11-10T09:25:55,769 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting a09dd52ec38f407dac5102174727338f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731230750520 2024-11-10T09:25:55,769 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 46018b55b8ff4bb892f193dca3d22ae3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230751682 2024-11-10T09:25:55,769 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d8a7d2c4804a60af63bc97e7454dee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230751682 2024-11-10T09:25:55,769 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b598b35706b4d1ab3df62062355ca58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230753848 2024-11-10T09:25:55,770 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a207a634eb4487af578be1e9c2564a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230753848 2024-11-10T09:25:55,776 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:55,778 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:55,778 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/3c3cea3ce9884d0a86b8c5c782965c4d is 50, key is test_row_0/B:col10/1731230753848/Put/seqid=0 2024-11-10T09:25:55,779 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411101f67a8b2f22f48dd990d5c6d0f0a2a7d_5619dfa0d7c1c633ebe4b8aecdc69e58 store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:55,781 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411101f67a8b2f22f48dd990d5c6d0f0a2a7d_5619dfa0d7c1c633ebe4b8aecdc69e58, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:55,781 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101f67a8b2f22f48dd990d5c6d0f0a2a7d_5619dfa0d7c1c633ebe4b8aecdc69e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:55,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742020_1196 (size=4469) 2024-11-10T09:25:55,797 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#A#compaction#164 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:55,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742019_1195 (size=12663) 2024-11-10T09:25:55,797 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/a6cc9444145548ce92cb3a1fc0d00dcd is 175, key is test_row_0/A:col10/1731230753848/Put/seqid=0 2024-11-10T09:25:55,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742021_1197 (size=31617) 2024-11-10T09:25:55,806 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/a6cc9444145548ce92cb3a1fc0d00dcd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a6cc9444145548ce92cb3a1fc0d00dcd 2024-11-10T09:25:55,811 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/A of 5619dfa0d7c1c633ebe4b8aecdc69e58 into a6cc9444145548ce92cb3a1fc0d00dcd(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:55,811 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:55,811 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/A, priority=13, startTime=1731230755767; duration=0sec 2024-11-10T09:25:55,812 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:55,812 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:A 2024-11-10T09:25:55,812 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:55,813 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:55,813 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/C is initiating minor compaction (all files) 2024-11-10T09:25:55,813 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/C in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,813 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/41eb7c424bce44e188b3a6dd09ad1352, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/3d5eee65e532436f959af8d513e478de, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/08bfc8f71a7e423ab4d202620fd11de0] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=36.0 K 2024-11-10T09:25:55,814 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41eb7c424bce44e188b3a6dd09ad1352, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731230750520 2024-11-10T09:25:55,814 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d5eee65e532436f959af8d513e478de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230751682 2024-11-10T09:25:55,815 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08bfc8f71a7e423ab4d202620fd11de0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230753848 2024-11-10T09:25:55,821 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#C#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:55,821 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/6d8864714a894210a11a11a7d918516b is 50, key is test_row_0/C:col10/1731230753848/Put/seqid=0 2024-11-10T09:25:55,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742022_1198 (size=12663) 2024-11-10T09:25:55,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T09:25:55,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:55,854 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-10T09:25:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b2820e4faeed4d9bbc1d7a105b83259a_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230754496/Put/seqid=0 2024-11-10T09:25:55,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742023_1199 (size=12304) 2024-11-10T09:25:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:55,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:55,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230815939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230816056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:25:56,202 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/3c3cea3ce9884d0a86b8c5c782965c4d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/3c3cea3ce9884d0a86b8c5c782965c4d 2024-11-10T09:25:56,207 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/B of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 3c3cea3ce9884d0a86b8c5c782965c4d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:56,207 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:56,207 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/B, priority=13, startTime=1731230755767; duration=0sec 2024-11-10T09:25:56,207 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:56,207 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:B 2024-11-10T09:25:56,231 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/6d8864714a894210a11a11a7d918516b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/6d8864714a894210a11a11a7d918516b 2024-11-10T09:25:56,236 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/C of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 6d8864714a894210a11a11a7d918516b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:56,236 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:56,236 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/C, priority=13, startTime=1731230755768; duration=0sec 2024-11-10T09:25:56,236 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:56,236 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:C 2024-11-10T09:25:56,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:56,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230816265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,270 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b2820e4faeed4d9bbc1d7a105b83259a_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2820e4faeed4d9bbc1d7a105b83259a_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:56,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/921e74ee33484facae249893655aca5f, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:56,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/921e74ee33484facae249893655aca5f is 175, key is test_row_0/A:col10/1731230754496/Put/seqid=0 2024-11-10T09:25:56,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742024_1200 (size=31105) 2024-11-10T09:25:56,278 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/921e74ee33484facae249893655aca5f 2024-11-10T09:25:56,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/0a91e975e2e9441ca53853e63d9322a1 is 50, key is test_row_0/B:col10/1731230754496/Put/seqid=0 2024-11-10T09:25:56,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742025_1201 (size=12151) 2024-11-10T09:25:56,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230816576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230816656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:56,657 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230816666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:56,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230816666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:56,669 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,670 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230816675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:56,677 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:56,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:56,697 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/0a91e975e2e9441ca53853e63d9322a1 2024-11-10T09:25:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/09103f174a574ffe97de755f1854bb79 is 50, key is test_row_0/C:col10/1731230754496/Put/seqid=0 2024-11-10T09:25:56,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742026_1202 (size=12151) 2024-11-10T09:25:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:25:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230817085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:57,108 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/09103f174a574ffe97de755f1854bb79 2024-11-10T09:25:57,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/921e74ee33484facae249893655aca5f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/921e74ee33484facae249893655aca5f 2024-11-10T09:25:57,115 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/921e74ee33484facae249893655aca5f, entries=150, sequenceid=235, filesize=30.4 K 2024-11-10T09:25:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/0a91e975e2e9441ca53853e63d9322a1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/0a91e975e2e9441ca53853e63d9322a1 2024-11-10T09:25:57,120 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/0a91e975e2e9441ca53853e63d9322a1, entries=150, sequenceid=235, filesize=11.9 K 2024-11-10T09:25:57,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/09103f174a574ffe97de755f1854bb79 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/09103f174a574ffe97de755f1854bb79 2024-11-10T09:25:57,125 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/09103f174a574ffe97de755f1854bb79, entries=150, sequenceid=235, filesize=11.9 K 2024-11-10T09:25:57,126 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1273ms, sequenceid=235, compaction requested=false 2024-11-10T09:25:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:57,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:57,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T09:25:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T09:25:57,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T09:25:57,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-10T09:25:57,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.2040 sec 2024-11-10T09:25:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:58,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:25:58,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-10T09:25:58,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111038d803a5ff594186aad86fca36893db6_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230755918/Put/seqid=0 2024-11-10T09:25:58,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742027_1203 (size=14844) 2024-11-10T09:25:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230818138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230818246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230818455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:58,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111038d803a5ff594186aad86fca36893db6_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111038d803a5ff594186aad86fca36893db6_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:58,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f68e032b89b148cbae014a044182a824, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:58,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f68e032b89b148cbae014a044182a824 is 175, key is test_row_0/A:col10/1731230755918/Put/seqid=0 2024-11-10T09:25:58,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742028_1204 (size=39799) 2024-11-10T09:25:58,525 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f68e032b89b148cbae014a044182a824 2024-11-10T09:25:58,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/b23192ddb5d14a67b196c65c379dfc45 is 50, key is test_row_0/B:col10/1731230755918/Put/seqid=0 2024-11-10T09:25:58,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742029_1205 (size=12201) 2024-11-10T09:25:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37522 deadline: 1731230818675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37524 deadline: 1731230818675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,677 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:58,677 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37512 deadline: 1731230818685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,687 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37496 deadline: 1731230818696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,697 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 5619dfa0d7c1c633ebe4b8aecdc69e58 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:25:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230818765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:58,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/b23192ddb5d14a67b196c65c379dfc45 2024-11-10T09:25:58,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/ca1d2753928b4dc29716a3d33e7123bf is 50, key is test_row_0/C:col10/1731230755918/Put/seqid=0 2024-11-10T09:25:58,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742030_1206 (size=12201) 2024-11-10T09:25:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T09:25:59,096 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:25:59,096 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:25:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-10T09:25:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:25:59,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:25:59,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:25:59,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:25:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:25:59,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:25:59,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:25:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:59,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:59,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:25:59,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:25:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230819275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:25:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:25:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:25:59,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/ca1d2753928b4dc29716a3d33e7123bf 2024-11-10T09:25:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/f68e032b89b148cbae014a044182a824 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f68e032b89b148cbae014a044182a824 2024-11-10T09:25:59,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f68e032b89b148cbae014a044182a824, entries=200, sequenceid=257, filesize=38.9 K 2024-11-10T09:25:59,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/b23192ddb5d14a67b196c65c379dfc45 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/b23192ddb5d14a67b196c65c379dfc45 2024-11-10T09:25:59,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/b23192ddb5d14a67b196c65c379dfc45, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T09:25:59,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/ca1d2753928b4dc29716a3d33e7123bf as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/ca1d2753928b4dc29716a3d33e7123bf 2024-11-10T09:25:59,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/ca1d2753928b4dc29716a3d33e7123bf, entries=150, sequenceid=257, filesize=11.9 K 2024-11-10T09:25:59,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1270ms, sequenceid=257, compaction requested=true 2024-11-10T09:25:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:25:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:59,367 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:25:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:25:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T09:25:59,368 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:59,368 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:59,368 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/A is initiating minor compaction (all files) 2024-11-10T09:25:59,369 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/A in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:59,369 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a6cc9444145548ce92cb3a1fc0d00dcd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/921e74ee33484facae249893655aca5f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f68e032b89b148cbae014a044182a824] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=100.1 K 2024-11-10T09:25:59,369 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:59,369 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:59,369 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/B is initiating minor compaction (all files) 2024-11-10T09:25:59,369 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a6cc9444145548ce92cb3a1fc0d00dcd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/921e74ee33484facae249893655aca5f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f68e032b89b148cbae014a044182a824] 2024-11-10T09:25:59,369 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/B in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:59,369 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/3c3cea3ce9884d0a86b8c5c782965c4d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/0a91e975e2e9441ca53853e63d9322a1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/b23192ddb5d14a67b196c65c379dfc45] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=36.1 K 2024-11-10T09:25:59,369 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6cc9444145548ce92cb3a1fc0d00dcd, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230753848 2024-11-10T09:25:59,370 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3cea3ce9884d0a86b8c5c782965c4d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230753848 2024-11-10T09:25:59,370 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 921e74ee33484facae249893655aca5f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230754496 2024-11-10T09:25:59,370 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a91e975e2e9441ca53853e63d9322a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230754496 2024-11-10T09:25:59,370 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f68e032b89b148cbae014a044182a824, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230755916 2024-11-10T09:25:59,370 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting b23192ddb5d14a67b196c65c379dfc45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230755916 2024-11-10T09:25:59,376 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:59,377 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#B#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:59,378 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/f247bcf0e2484ca8b32f4ac7a6fc5f24 is 50, key is test_row_0/B:col10/1731230755918/Put/seqid=0 2024-11-10T09:25:59,381 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100f79ca96438d4cc0ad1fdffe8cd1635a_5619dfa0d7c1c633ebe4b8aecdc69e58 store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:59,383 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100f79ca96438d4cc0ad1fdffe8cd1635a_5619dfa0d7c1c633ebe4b8aecdc69e58, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:59,383 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100f79ca96438d4cc0ad1fdffe8cd1635a_5619dfa0d7c1c633ebe4b8aecdc69e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:59,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742031_1207 (size=12815) 2024-11-10T09:25:59,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742032_1208 (size=4469) 2024-11-10T09:25:59,392 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/f247bcf0e2484ca8b32f4ac7a6fc5f24 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f247bcf0e2484ca8b32f4ac7a6fc5f24 2024-11-10T09:25:59,396 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/B of 5619dfa0d7c1c633ebe4b8aecdc69e58 into f247bcf0e2484ca8b32f4ac7a6fc5f24(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:59,396 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:59,396 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/B, priority=13, startTime=1731230759367; duration=0sec 2024-11-10T09:25:59,397 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:25:59,397 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:B 2024-11-10T09:25:59,397 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:25:59,398 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:25:59,398 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/C is initiating minor compaction (all files) 2024-11-10T09:25:59,398 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/C in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:59,398 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/6d8864714a894210a11a11a7d918516b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/09103f174a574ffe97de755f1854bb79, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/ca1d2753928b4dc29716a3d33e7123bf] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=36.1 K 2024-11-10T09:25:59,398 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d8864714a894210a11a11a7d918516b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230753848 2024-11-10T09:25:59,398 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 09103f174a574ffe97de755f1854bb79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230754496 2024-11-10T09:25:59,399 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1d2753928b4dc29716a3d33e7123bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230755916 2024-11-10T09:25:59,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T09:25:59,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:25:59,404 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T09:25:59,405 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#C#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:59,406 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/97a50aaff29546199da34ce983c08c2e is 50, key is test_row_0/C:col10/1731230755918/Put/seqid=0 2024-11-10T09:25:59,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110832380e1524b4629a570984115968839_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230758136/Put/seqid=0 2024-11-10T09:25:59,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742033_1209 (size=12815) 2024-11-10T09:25:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:25:59,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742034_1210 (size=12454) 2024-11-10T09:25:59,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:25:59,426 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110832380e1524b4629a570984115968839_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110832380e1524b4629a570984115968839_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:25:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/03ea716ad8154102969d32963ce0ef65, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:25:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/03ea716ad8154102969d32963ce0ef65 is 175, key is test_row_0/A:col10/1731230758136/Put/seqid=0 2024-11-10T09:25:59,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742035_1211 (size=31255) 2024-11-10T09:25:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:25:59,788 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#A#compaction#174 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:25:59,789 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/db121340ade64da0aef209f7bb93d91a is 175, key is test_row_0/A:col10/1731230755918/Put/seqid=0 2024-11-10T09:25:59,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742036_1212 (size=31769) 2024-11-10T09:25:59,800 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/db121340ade64da0aef209f7bb93d91a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/db121340ade64da0aef209f7bb93d91a 2024-11-10T09:25:59,805 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/A of 5619dfa0d7c1c633ebe4b8aecdc69e58 into db121340ade64da0aef209f7bb93d91a(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:59,805 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:59,806 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/A, priority=13, startTime=1731230759367; duration=0sec 2024-11-10T09:25:59,806 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:59,806 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:A 2024-11-10T09:25:59,820 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/97a50aaff29546199da34ce983c08c2e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/97a50aaff29546199da34ce983c08c2e 2024-11-10T09:25:59,825 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/C of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 97a50aaff29546199da34ce983c08c2e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:25:59,825 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:25:59,826 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/C, priority=13, startTime=1731230759367; duration=0sec 2024-11-10T09:25:59,826 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:25:59,826 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:C 2024-11-10T09:25:59,831 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/03ea716ad8154102969d32963ce0ef65 2024-11-10T09:25:59,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/f36b5e30272e4401b940678a9fc8bbb1 is 50, key is test_row_0/B:col10/1731230758136/Put/seqid=0 2024-11-10T09:25:59,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742037_1213 (size=12301) 2024-11-10T09:26:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:26:00,256 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/f36b5e30272e4401b940678a9fc8bbb1 2024-11-10T09:26:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:00,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/f0466bc0fce34b89a750d8cca9170331 is 50, key is test_row_0/C:col10/1731230758136/Put/seqid=0 2024-11-10T09:26:00,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742038_1214 (size=12301) 2024-11-10T09:26:00,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230820376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230820486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:00,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230820695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:00,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:00,725 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/f0466bc0fce34b89a750d8cca9170331 2024-11-10T09:26:00,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/03ea716ad8154102969d32963ce0ef65 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/03ea716ad8154102969d32963ce0ef65 2024-11-10T09:26:00,734 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/03ea716ad8154102969d32963ce0ef65, entries=150, sequenceid=274, filesize=30.5 K 2024-11-10T09:26:00,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/f36b5e30272e4401b940678a9fc8bbb1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f36b5e30272e4401b940678a9fc8bbb1 2024-11-10T09:26:00,739 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f36b5e30272e4401b940678a9fc8bbb1, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T09:26:00,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/f0466bc0fce34b89a750d8cca9170331 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/f0466bc0fce34b89a750d8cca9170331 2024-11-10T09:26:00,747 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/f0466bc0fce34b89a750d8cca9170331, entries=150, sequenceid=274, filesize=12.0 K 2024-11-10T09:26:00,748 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1345ms, sequenceid=274, compaction requested=false 2024-11-10T09:26:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:26:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:00,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-10T09:26:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-10T09:26:00,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T09:26:00,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6510 sec 2024-11-10T09:26:00,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.6550 sec 2024-11-10T09:26:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:01,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:01,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T09:26:01,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110967fe2b69f45460b9251ec6004d48de8_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230760362/Put/seqid=0 2024-11-10T09:26:01,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742039_1215 (size=14994) 2024-11-10T09:26:01,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230821050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:01,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230821156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T09:26:01,246 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:01,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-10T09:26:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:01,248 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:01,248 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:01,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:01,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:01,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230821366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:01,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:01,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110967fe2b69f45460b9251ec6004d48de8_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110967fe2b69f45460b9251ec6004d48de8_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:01,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/cbd7d5c70d8a446e8f5ddb36db5c97e6, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:26:01,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/cbd7d5c70d8a446e8f5ddb36db5c97e6 is 175, key is test_row_0/A:col10/1731230760362/Put/seqid=0 2024-11-10T09:26:01,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742040_1216 (size=39949) 2024-11-10T09:26:01,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230821675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:01,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,832 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/cbd7d5c70d8a446e8f5ddb36db5c97e6 2024-11-10T09:26:01,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/41549c67d91a4824a656082f8f5756f5 is 50, key is test_row_0/B:col10/1731230760362/Put/seqid=0 2024-11-10T09:26:01,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742041_1217 (size=12301) 2024-11-10T09:26:01,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:01,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:01,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:01,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:02,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:02,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,146 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:26:02,147 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-10T09:26:02,147 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,147 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,148 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,149 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-10T09:26:02,149 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,149 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,149 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,149 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-10T09:26:02,149 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,149 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,150 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,156 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-10T09:26:02,156 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,156 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,156 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37568 deadline: 1731230822185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:02,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/41549c67d91a4824a656082f8f5756f5 2024-11-10T09:26:02,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/dfed4139c33a4c0fb1c8bc00887e5a3a is 50, key is test_row_0/C:col10/1731230760362/Put/seqid=0 2024-11-10T09:26:02,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742042_1218 (size=12301) 2024-11-10T09:26:02,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:02,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:02,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/dfed4139c33a4c0fb1c8bc00887e5a3a 2024-11-10T09:26:02,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/cbd7d5c70d8a446e8f5ddb36db5c97e6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/cbd7d5c70d8a446e8f5ddb36db5c97e6 2024-11-10T09:26:02,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/cbd7d5c70d8a446e8f5ddb36db5c97e6, entries=200, sequenceid=297, filesize=39.0 K 2024-11-10T09:26:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/41549c67d91a4824a656082f8f5756f5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/41549c67d91a4824a656082f8f5756f5 2024-11-10T09:26:02,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/41549c67d91a4824a656082f8f5756f5, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T09:26:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/dfed4139c33a4c0fb1c8bc00887e5a3a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/dfed4139c33a4c0fb1c8bc00887e5a3a 2024-11-10T09:26:02,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/dfed4139c33a4c0fb1c8bc00887e5a3a, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T09:26:02,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1679ms, sequenceid=297, compaction requested=true 2024-11-10T09:26:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:26:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:02,688 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:02,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:02,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:02,689 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:02,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5619dfa0d7c1c633ebe4b8aecdc69e58:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:02,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:02,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:02,690 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:02,690 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/B is initiating minor compaction (all files) 2024-11-10T09:26:02,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/A is initiating minor compaction (all files) 2024-11-10T09:26:02,690 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/A in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,690 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/B in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,690 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/db121340ade64da0aef209f7bb93d91a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/03ea716ad8154102969d32963ce0ef65, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/cbd7d5c70d8a446e8f5ddb36db5c97e6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=100.6 K 2024-11-10T09:26:02,690 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f247bcf0e2484ca8b32f4ac7a6fc5f24, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f36b5e30272e4401b940678a9fc8bbb1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/41549c67d91a4824a656082f8f5756f5] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=36.5 K 2024-11-10T09:26:02,690 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/db121340ade64da0aef209f7bb93d91a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/03ea716ad8154102969d32963ce0ef65, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/cbd7d5c70d8a446e8f5ddb36db5c97e6] 2024-11-10T09:26:02,690 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f247bcf0e2484ca8b32f4ac7a6fc5f24, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230755916 2024-11-10T09:26:02,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting db121340ade64da0aef209f7bb93d91a, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230755916 2024-11-10T09:26:02,691 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36b5e30272e4401b940678a9fc8bbb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731230758116 2024-11-10T09:26:02,691 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ea716ad8154102969d32963ce0ef65, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731230758116 2024-11-10T09:26:02,691 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41549c67d91a4824a656082f8f5756f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731230760362 2024-11-10T09:26:02,691 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd7d5c70d8a446e8f5ddb36db5c97e6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731230760362 2024-11-10T09:26:02,701 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#B#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:02,701 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/d2eca9cdd2694acfbc9cfc44e90185c3 is 50, key is test_row_0/B:col10/1731230760362/Put/seqid=0 2024-11-10T09:26:02,702 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:26:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:02,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:02,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-10T09:26:02,703 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-10T09:26:02,703 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,703 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,703 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,705 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d3f6b8e10c16422fae34fd2b473a5e8f_5619dfa0d7c1c633ebe4b8aecdc69e58 store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:26:02,709 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-10T09:26:02,709 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,709 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,709 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742043_1219 (size=13017) 2024-11-10T09:26:02,719 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/d2eca9cdd2694acfbc9cfc44e90185c3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/d2eca9cdd2694acfbc9cfc44e90185c3 2024-11-10T09:26:02,723 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-10T09:26:02,724 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,724 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,724 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108a4938655db4414b85b4d3c5cddd03ad_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230761049/Put/seqid=0 2024-11-10T09:26:02,731 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/B of 5619dfa0d7c1c633ebe4b8aecdc69e58 into d2eca9cdd2694acfbc9cfc44e90185c3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:02,731 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:26:02,731 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/B, priority=13, startTime=1731230762688; duration=0sec 2024-11-10T09:26:02,731 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:02,731 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:B 2024-11-10T09:26:02,731 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:02,732 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:02,733 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 5619dfa0d7c1c633ebe4b8aecdc69e58/C is initiating minor compaction (all files) 2024-11-10T09:26:02,733 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5619dfa0d7c1c633ebe4b8aecdc69e58/C in TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,733 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/97a50aaff29546199da34ce983c08c2e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/f0466bc0fce34b89a750d8cca9170331, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/dfed4139c33a4c0fb1c8bc00887e5a3a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp, totalSize=36.5 K 2024-11-10T09:26:02,733 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97a50aaff29546199da34ce983c08c2e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731230755916 2024-11-10T09:26:02,733 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0466bc0fce34b89a750d8cca9170331, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731230758116 2024-11-10T09:26:02,734 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfed4139c33a4c0fb1c8bc00887e5a3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731230760362 2024-11-10T09:26:02,742 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d3f6b8e10c16422fae34fd2b473a5e8f_5619dfa0d7c1c633ebe4b8aecdc69e58, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:26:02,743 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d3f6b8e10c16422fae34fd2b473a5e8f_5619dfa0d7c1c633ebe4b8aecdc69e58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:26:02,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742044_1220 (size=12454) 2024-11-10T09:26:02,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:02,751 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-10T09:26:02,751 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:02,751 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742045_1221 (size=4469) 2024-11-10T09:26:02,751 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:02,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:02,752 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#C#compaction#185 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:02,753 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/49ddb659b302462eb76bb3d8e215d865 is 50, key is test_row_0/C:col10/1731230760362/Put/seqid=0 2024-11-10T09:26:02,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108a4938655db4414b85b4d3c5cddd03ad_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108a4938655db4414b85b4d3c5cddd03ad_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:02,755 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/1f959c9af82549d99c5b5411c4ac8fc8, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:26:02,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/1f959c9af82549d99c5b5411c4ac8fc8 is 175, key is test_row_0/A:col10/1731230761049/Put/seqid=0 2024-11-10T09:26:02,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742046_1222 (size=13017) 2024-11-10T09:26:02,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742047_1223 (size=31255) 2024-11-10T09:26:02,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:02,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:02,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:02,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,152 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5619dfa0d7c1c633ebe4b8aecdc69e58#A#compaction#183 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:03,154 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/6610a955e91d44bca2f245bf6a085de2 is 175, key is test_row_0/A:col10/1731230760362/Put/seqid=0 2024-11-10T09:26:03,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/1f959c9af82549d99c5b5411c4ac8fc8 2024-11-10T09:26:03,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742048_1224 (size=31971) 2024-11-10T09:26:03,178 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/49ddb659b302462eb76bb3d8e215d865 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/49ddb659b302462eb76bb3d8e215d865 2024-11-10T09:26:03,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/338fb887bd4f4d02865ef9adccf4d0fd is 50, key is test_row_0/B:col10/1731230761049/Put/seqid=0 2024-11-10T09:26:03,183 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/C of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 49ddb659b302462eb76bb3d8e215d865(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:03,183 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:26:03,184 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/C, priority=13, startTime=1731230762689; duration=0sec 2024-11-10T09:26:03,184 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:03,184 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:C 2024-11-10T09:26:03,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742049_1225 (size=12301) 2024-11-10T09:26:03,197 INFO [Thread-669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-669. 2024-11-10T09:26:03,197 DEBUG [Thread-669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:03,197 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:03,197 DEBUG [Thread-669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:03,197 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:03,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:03,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:03,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:03,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:03,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/338fb887bd4f4d02865ef9adccf4d0fd 2024-11-10T09:26:03,587 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/6610a955e91d44bca2f245bf6a085de2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/6610a955e91d44bca2f245bf6a085de2 2024-11-10T09:26:03,593 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5619dfa0d7c1c633ebe4b8aecdc69e58/A of 5619dfa0d7c1c633ebe4b8aecdc69e58 into 6610a955e91d44bca2f245bf6a085de2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:03,593 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:26:03,593 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58., storeName=5619dfa0d7c1c633ebe4b8aecdc69e58/A, priority=13, startTime=1731230762688; duration=0sec 2024-11-10T09:26:03,594 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:03,594 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5619dfa0d7c1c633ebe4b8aecdc69e58:A 2024-11-10T09:26:03,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/84d8a30f3f854167a86e595a4d0a83a2 is 50, key is test_row_0/C:col10/1731230761049/Put/seqid=0 2024-11-10T09:26:03,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742050_1226 (size=12301) 2024-11-10T09:26:03,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:03,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:03,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:03,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:04,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/84d8a30f3f854167a86e595a4d0a83a2 2024-11-10T09:26:04,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/1f959c9af82549d99c5b5411c4ac8fc8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/1f959c9af82549d99c5b5411c4ac8fc8 2024-11-10T09:26:04,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/1f959c9af82549d99c5b5411c4ac8fc8, entries=150, sequenceid=313, filesize=30.5 K 2024-11-10T09:26:04,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/338fb887bd4f4d02865ef9adccf4d0fd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/338fb887bd4f4d02865ef9adccf4d0fd 2024-11-10T09:26:04,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/338fb887bd4f4d02865ef9adccf4d0fd, entries=150, sequenceid=313, filesize=12.0 K 2024-11-10T09:26:04,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/84d8a30f3f854167a86e595a4d0a83a2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/84d8a30f3f854167a86e595a4d0a83a2 2024-11-10T09:26:04,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:04,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:04,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. as already flushing 2024-11-10T09:26:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:04,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:04,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:04,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/84d8a30f3f854167a86e595a4d0a83a2, entries=150, sequenceid=313, filesize=12.0 K 2024-11-10T09:26:04,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=26.84 KB/27480 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1330ms, sequenceid=313, compaction requested=false 2024-11-10T09:26:04,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:26:04,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T09:26:04,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:04,181 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 5619dfa0d7c1c633ebe4b8aecdc69e58 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T09:26:04,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111039c80daaa6ee49d98c2a5b22cfbfe886_5619dfa0d7c1c633ebe4b8aecdc69e58 is 50, key is test_row_0/A:col10/1731230763196/Put/seqid=0 2024-11-10T09:26:04,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742051_1227 (size=9914) 2024-11-10T09:26:04,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:04,611 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111039c80daaa6ee49d98c2a5b22cfbfe886_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039c80daaa6ee49d98c2a5b22cfbfe886_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/c77344dae87844b4a59bebf592bf70b0, store: [table=TestAcidGuarantees family=A region=5619dfa0d7c1c633ebe4b8aecdc69e58] 2024-11-10T09:26:04,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/c77344dae87844b4a59bebf592bf70b0 is 175, key is test_row_0/A:col10/1731230763196/Put/seqid=0 2024-11-10T09:26:04,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742052_1228 (size=22561) 2024-11-10T09:26:05,019 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/c77344dae87844b4a59bebf592bf70b0 2024-11-10T09:26:05,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/d54f619c99844e9a8286ee28fd96bbc8 is 50, key is test_row_0/B:col10/1731230763196/Put/seqid=0 2024-11-10T09:26:05,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742053_1229 (size=9857) 2024-11-10T09:26:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:05,439 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/d54f619c99844e9a8286ee28fd96bbc8 2024-11-10T09:26:05,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/7248208397424fb383da112a65a50faf is 50, key is test_row_0/C:col10/1731230763196/Put/seqid=0 2024-11-10T09:26:05,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742054_1230 (size=9857) 2024-11-10T09:26:05,906 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/7248208397424fb383da112a65a50faf 2024-11-10T09:26:05,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/A/c77344dae87844b4a59bebf592bf70b0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/c77344dae87844b4a59bebf592bf70b0 2024-11-10T09:26:05,921 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/c77344dae87844b4a59bebf592bf70b0, entries=100, sequenceid=323, filesize=22.0 K 2024-11-10T09:26:05,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/B/d54f619c99844e9a8286ee28fd96bbc8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/d54f619c99844e9a8286ee28fd96bbc8 2024-11-10T09:26:05,926 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/d54f619c99844e9a8286ee28fd96bbc8, entries=100, sequenceid=323, filesize=9.6 K 2024-11-10T09:26:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/.tmp/C/7248208397424fb383da112a65a50faf as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7248208397424fb383da112a65a50faf 2024-11-10T09:26:05,930 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7248208397424fb383da112a65a50faf, entries=100, sequenceid=323, filesize=9.6 K 2024-11-10T09:26:05,931 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5619dfa0d7c1c633ebe4b8aecdc69e58 in 1750ms, sequenceid=323, compaction requested=true 2024-11-10T09:26:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: 2024-11-10T09:26:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:05,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-10T09:26:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-10T09:26:05,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T09:26:05,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.6840 sec 2024-11-10T09:26:05,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 4.6880 sec 2024-11-10T09:26:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T09:26:09,448 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4280 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4031 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1757 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5269 rows 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1749 2024-11-10T09:26:09,448 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5246 rows 2024-11-10T09:26:09,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:26:09,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:26:09,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:09,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:09,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:09,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:26:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T09:26:09,456 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230769456"}]},"ts":"1731230769456"} 2024-11-10T09:26:09,458 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:26:09,459 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:26:09,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:26:09,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, UNASSIGN}] 2024-11-10T09:26:09,463 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, UNASSIGN 2024-11-10T09:26:09,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:09,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, UNASSIGN because future has completed 2024-11-10T09:26:09,467 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:26:09,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:26:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T09:26:09,619 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,619 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:26:09,620 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 5619dfa0d7c1c633ebe4b8aecdc69e58, disabling compactions & flushes 2024-11-10T09:26:09,620 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:09,620 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:09,620 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. after waiting 0 ms 2024-11-10T09:26:09,620 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:09,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/268d94727385427b8f5382e5260f47c8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f15f6decd81a4e719adc27377dcdaf56, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d25b4006411e4521ad4a9226ea6e1104, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/29e3b85f23bf401886053f73a71e3dbf, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/2f07b3d7dbfa40c39963d9d5bafab4e5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e81c1215822a4affbee4c710ac3ff440, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/7f47ec181864479e8a93c28b38db2be4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e1c5cec4652b40e289410cf30cab4311, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d77a91ef82964e45b37b42e6af549ccd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/3cb18c50d1ef47958ebac827db4abbeb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/02dbf54792c441af91ae53082abcd69c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/daf955e5318d48e68f12e8914f039a9c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a09dd52ec38f407dac5102174727338f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/64d8a7d2c4804a60af63bc97e7454dee, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d5a207a634eb4487af578be1e9c2564a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a6cc9444145548ce92cb3a1fc0d00dcd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/921e74ee33484facae249893655aca5f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f68e032b89b148cbae014a044182a824, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/db121340ade64da0aef209f7bb93d91a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/03ea716ad8154102969d32963ce0ef65, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/cbd7d5c70d8a446e8f5ddb36db5c97e6] to archive 2024-11-10T09:26:09,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:26:09,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/268d94727385427b8f5382e5260f47c8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/268d94727385427b8f5382e5260f47c8 2024-11-10T09:26:09,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f15f6decd81a4e719adc27377dcdaf56 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f15f6decd81a4e719adc27377dcdaf56 2024-11-10T09:26:09,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d25b4006411e4521ad4a9226ea6e1104 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d25b4006411e4521ad4a9226ea6e1104 2024-11-10T09:26:09,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/29e3b85f23bf401886053f73a71e3dbf to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/29e3b85f23bf401886053f73a71e3dbf 2024-11-10T09:26:09,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/2f07b3d7dbfa40c39963d9d5bafab4e5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/2f07b3d7dbfa40c39963d9d5bafab4e5 2024-11-10T09:26:09,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e81c1215822a4affbee4c710ac3ff440 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e81c1215822a4affbee4c710ac3ff440 2024-11-10T09:26:09,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/7f47ec181864479e8a93c28b38db2be4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/7f47ec181864479e8a93c28b38db2be4 2024-11-10T09:26:09,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e1c5cec4652b40e289410cf30cab4311 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/e1c5cec4652b40e289410cf30cab4311 2024-11-10T09:26:09,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d77a91ef82964e45b37b42e6af549ccd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d77a91ef82964e45b37b42e6af549ccd 2024-11-10T09:26:09,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/3cb18c50d1ef47958ebac827db4abbeb to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/3cb18c50d1ef47958ebac827db4abbeb 2024-11-10T09:26:09,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/02dbf54792c441af91ae53082abcd69c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/02dbf54792c441af91ae53082abcd69c 2024-11-10T09:26:09,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/daf955e5318d48e68f12e8914f039a9c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/daf955e5318d48e68f12e8914f039a9c 2024-11-10T09:26:09,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a09dd52ec38f407dac5102174727338f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a09dd52ec38f407dac5102174727338f 2024-11-10T09:26:09,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/64d8a7d2c4804a60af63bc97e7454dee to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/64d8a7d2c4804a60af63bc97e7454dee 2024-11-10T09:26:09,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d5a207a634eb4487af578be1e9c2564a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/d5a207a634eb4487af578be1e9c2564a 2024-11-10T09:26:09,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a6cc9444145548ce92cb3a1fc0d00dcd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/a6cc9444145548ce92cb3a1fc0d00dcd 2024-11-10T09:26:09,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/921e74ee33484facae249893655aca5f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/921e74ee33484facae249893655aca5f 2024-11-10T09:26:09,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f68e032b89b148cbae014a044182a824 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/f68e032b89b148cbae014a044182a824 2024-11-10T09:26:09,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/db121340ade64da0aef209f7bb93d91a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/db121340ade64da0aef209f7bb93d91a 2024-11-10T09:26:09,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/03ea716ad8154102969d32963ce0ef65 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/03ea716ad8154102969d32963ce0ef65 2024-11-10T09:26:09,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/cbd7d5c70d8a446e8f5ddb36db5c97e6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/cbd7d5c70d8a446e8f5ddb36db5c97e6 2024-11-10T09:26:09,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7cb8f43e5ed94ea48c30f34861412181, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7fc9fa869c934587aa52ece9cb48be77, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f8806765fde94c268cdd3576f90bcba5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/046618aff0504b85bef816e9bdd53171, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/36411f4a0e31457b9ddcf254e4b3e5e8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/925e0218f6f54372996012cd6d3cfd05, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/1ea1efca0036482c9de97f21875c4c0b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/db143fbf55ac43358550ef57f34c6618, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/bab9a2997423438d8fa90bb08037be16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/77949c0d05fb4bd2a2c5293a8f75ca9f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/85fe951ff7e94762b57fbc5040fd019c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/ca2ec49a661e46e097c4e048d3be5ff1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/265e7c0fe30446feb132e1951af9ef7e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/46018b55b8ff4bb892f193dca3d22ae3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/3c3cea3ce9884d0a86b8c5c782965c4d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/5b598b35706b4d1ab3df62062355ca58, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/0a91e975e2e9441ca53853e63d9322a1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f247bcf0e2484ca8b32f4ac7a6fc5f24, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/b23192ddb5d14a67b196c65c379dfc45, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f36b5e30272e4401b940678a9fc8bbb1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/41549c67d91a4824a656082f8f5756f5] to archive 2024-11-10T09:26:09,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:26:09,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7cb8f43e5ed94ea48c30f34861412181 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7cb8f43e5ed94ea48c30f34861412181 2024-11-10T09:26:09,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7fc9fa869c934587aa52ece9cb48be77 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/7fc9fa869c934587aa52ece9cb48be77 2024-11-10T09:26:09,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f8806765fde94c268cdd3576f90bcba5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f8806765fde94c268cdd3576f90bcba5 2024-11-10T09:26:09,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/046618aff0504b85bef816e9bdd53171 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/046618aff0504b85bef816e9bdd53171 2024-11-10T09:26:09,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/36411f4a0e31457b9ddcf254e4b3e5e8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/36411f4a0e31457b9ddcf254e4b3e5e8 2024-11-10T09:26:09,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/925e0218f6f54372996012cd6d3cfd05 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/925e0218f6f54372996012cd6d3cfd05 2024-11-10T09:26:09,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/1ea1efca0036482c9de97f21875c4c0b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/1ea1efca0036482c9de97f21875c4c0b 2024-11-10T09:26:09,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/db143fbf55ac43358550ef57f34c6618 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/db143fbf55ac43358550ef57f34c6618 2024-11-10T09:26:09,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/bab9a2997423438d8fa90bb08037be16 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/bab9a2997423438d8fa90bb08037be16 2024-11-10T09:26:09,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/77949c0d05fb4bd2a2c5293a8f75ca9f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/77949c0d05fb4bd2a2c5293a8f75ca9f 2024-11-10T09:26:09,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/85fe951ff7e94762b57fbc5040fd019c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/85fe951ff7e94762b57fbc5040fd019c 2024-11-10T09:26:09,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/ca2ec49a661e46e097c4e048d3be5ff1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/ca2ec49a661e46e097c4e048d3be5ff1 2024-11-10T09:26:09,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/265e7c0fe30446feb132e1951af9ef7e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/265e7c0fe30446feb132e1951af9ef7e 2024-11-10T09:26:09,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/46018b55b8ff4bb892f193dca3d22ae3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/46018b55b8ff4bb892f193dca3d22ae3 2024-11-10T09:26:09,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/3c3cea3ce9884d0a86b8c5c782965c4d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/3c3cea3ce9884d0a86b8c5c782965c4d 2024-11-10T09:26:09,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/5b598b35706b4d1ab3df62062355ca58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/5b598b35706b4d1ab3df62062355ca58 2024-11-10T09:26:09,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/0a91e975e2e9441ca53853e63d9322a1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/0a91e975e2e9441ca53853e63d9322a1 2024-11-10T09:26:09,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f247bcf0e2484ca8b32f4ac7a6fc5f24 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f247bcf0e2484ca8b32f4ac7a6fc5f24 2024-11-10T09:26:09,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/b23192ddb5d14a67b196c65c379dfc45 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/b23192ddb5d14a67b196c65c379dfc45 2024-11-10T09:26:09,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f36b5e30272e4401b940678a9fc8bbb1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/f36b5e30272e4401b940678a9fc8bbb1 2024-11-10T09:26:09,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/41549c67d91a4824a656082f8f5756f5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/41549c67d91a4824a656082f8f5756f5 2024-11-10T09:26:09,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/9e226aafb535478a817e08a496e87e49, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/c190e43fe3c44d1abe253b7d5882f981, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/a6a50fed2ca04cc5b10bf190e6863c2a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8bd898ca0a4a4f94b72941ea46921cad, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/1a838e27445b4ca482aed40bb6821224, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/e23d00d1b6b74f8cae4a23e79834468b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/10417dbcfc69403bb55dba9f9bf74a64, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/fb06a5d2fa554486a53d913ef37a653d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7f5a47555a2a40eeb1ab40167264c5b6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/29385bbc400448de8994b4c7e914a5e7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/d890755d1e6440258a5cec42e5e45295, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/41eb7c424bce44e188b3a6dd09ad1352, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8e02cb8cee1a4fb39cd33667b00bbdb1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/3d5eee65e532436f959af8d513e478de, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/6d8864714a894210a11a11a7d918516b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/08bfc8f71a7e423ab4d202620fd11de0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/09103f174a574ffe97de755f1854bb79, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/97a50aaff29546199da34ce983c08c2e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/ca1d2753928b4dc29716a3d33e7123bf, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/f0466bc0fce34b89a750d8cca9170331, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/dfed4139c33a4c0fb1c8bc00887e5a3a] to archive 2024-11-10T09:26:09,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:26:09,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/9e226aafb535478a817e08a496e87e49 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/9e226aafb535478a817e08a496e87e49 2024-11-10T09:26:09,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/c190e43fe3c44d1abe253b7d5882f981 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/c190e43fe3c44d1abe253b7d5882f981 2024-11-10T09:26:09,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/a6a50fed2ca04cc5b10bf190e6863c2a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/a6a50fed2ca04cc5b10bf190e6863c2a 2024-11-10T09:26:09,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8bd898ca0a4a4f94b72941ea46921cad to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8bd898ca0a4a4f94b72941ea46921cad 2024-11-10T09:26:09,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/1a838e27445b4ca482aed40bb6821224 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/1a838e27445b4ca482aed40bb6821224 2024-11-10T09:26:09,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/e23d00d1b6b74f8cae4a23e79834468b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/e23d00d1b6b74f8cae4a23e79834468b 2024-11-10T09:26:09,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/10417dbcfc69403bb55dba9f9bf74a64 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/10417dbcfc69403bb55dba9f9bf74a64 2024-11-10T09:26:09,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/fb06a5d2fa554486a53d913ef37a653d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/fb06a5d2fa554486a53d913ef37a653d 2024-11-10T09:26:09,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7f5a47555a2a40eeb1ab40167264c5b6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7f5a47555a2a40eeb1ab40167264c5b6 2024-11-10T09:26:09,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/29385bbc400448de8994b4c7e914a5e7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/29385bbc400448de8994b4c7e914a5e7 2024-11-10T09:26:09,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/d890755d1e6440258a5cec42e5e45295 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/d890755d1e6440258a5cec42e5e45295 2024-11-10T09:26:09,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/41eb7c424bce44e188b3a6dd09ad1352 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/41eb7c424bce44e188b3a6dd09ad1352 2024-11-10T09:26:09,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8e02cb8cee1a4fb39cd33667b00bbdb1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/8e02cb8cee1a4fb39cd33667b00bbdb1 2024-11-10T09:26:09,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/3d5eee65e532436f959af8d513e478de to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/3d5eee65e532436f959af8d513e478de 2024-11-10T09:26:09,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/6d8864714a894210a11a11a7d918516b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/6d8864714a894210a11a11a7d918516b 2024-11-10T09:26:09,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/08bfc8f71a7e423ab4d202620fd11de0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/08bfc8f71a7e423ab4d202620fd11de0 2024-11-10T09:26:09,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/09103f174a574ffe97de755f1854bb79 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/09103f174a574ffe97de755f1854bb79 2024-11-10T09:26:09,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/97a50aaff29546199da34ce983c08c2e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/97a50aaff29546199da34ce983c08c2e 2024-11-10T09:26:09,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/ca1d2753928b4dc29716a3d33e7123bf to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/ca1d2753928b4dc29716a3d33e7123bf 2024-11-10T09:26:09,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/f0466bc0fce34b89a750d8cca9170331 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/f0466bc0fce34b89a750d8cca9170331 2024-11-10T09:26:09,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/dfed4139c33a4c0fb1c8bc00887e5a3a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/dfed4139c33a4c0fb1c8bc00887e5a3a 2024-11-10T09:26:09,687 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-11-10T09:26:09,688 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58. 2024-11-10T09:26:09,688 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 5619dfa0d7c1c633ebe4b8aecdc69e58: Waiting for close lock at 1731230769620Running coprocessor pre-close hooks at 1731230769620Disabling compacts and flushes for region at 1731230769620Disabling writes for close at 1731230769620Writing region close event to WAL at 1731230769684 (+64 ms)Running coprocessor post-close hooks at 1731230769688 (+4 ms)Closed at 1731230769688 2024-11-10T09:26:09,690 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,690 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=5619dfa0d7c1c633ebe4b8aecdc69e58, regionState=CLOSED 2024-11-10T09:26:09,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:26:09,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T09:26:09,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 5619dfa0d7c1c633ebe4b8aecdc69e58, server=802346a334f5,32883,1731230704436 in 225 msec 2024-11-10T09:26:09,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-10T09:26:09,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5619dfa0d7c1c633ebe4b8aecdc69e58, UNASSIGN in 232 msec 2024-11-10T09:26:09,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T09:26:09,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 236 msec 2024-11-10T09:26:09,698 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230769698"}]},"ts":"1731230769698"} 2024-11-10T09:26:09,699 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:26:09,699 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:26:09,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 249 msec 2024-11-10T09:26:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T09:26:09,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:09,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:26:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,778 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T09:26:09,780 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,784 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,788 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/recovered.edits] 2024-11-10T09:26:09,793 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/1f959c9af82549d99c5b5411c4ac8fc8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/1f959c9af82549d99c5b5411c4ac8fc8 2024-11-10T09:26:09,795 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/6610a955e91d44bca2f245bf6a085de2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/6610a955e91d44bca2f245bf6a085de2 2024-11-10T09:26:09,797 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/c77344dae87844b4a59bebf592bf70b0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/A/c77344dae87844b4a59bebf592bf70b0 2024-11-10T09:26:09,800 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/338fb887bd4f4d02865ef9adccf4d0fd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/338fb887bd4f4d02865ef9adccf4d0fd 2024-11-10T09:26:09,802 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/d2eca9cdd2694acfbc9cfc44e90185c3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/d2eca9cdd2694acfbc9cfc44e90185c3 2024-11-10T09:26:09,803 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/d54f619c99844e9a8286ee28fd96bbc8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/B/d54f619c99844e9a8286ee28fd96bbc8 2024-11-10T09:26:09,807 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/49ddb659b302462eb76bb3d8e215d865 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/49ddb659b302462eb76bb3d8e215d865 2024-11-10T09:26:09,809 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7248208397424fb383da112a65a50faf to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/7248208397424fb383da112a65a50faf 2024-11-10T09:26:09,812 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/84d8a30f3f854167a86e595a4d0a83a2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/C/84d8a30f3f854167a86e595a4d0a83a2 2024-11-10T09:26:09,815 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/recovered.edits/326.seqid to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58/recovered.edits/326.seqid 2024-11-10T09:26:09,816 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,816 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:26:09,816 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:26:09,817 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T09:26:09,821 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100acec475dfb84181a97c812e1bd3b942_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100acec475dfb84181a97c812e1bd3b942_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,822 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111010df65902baf444cb588fd25cb902c92_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111010df65902baf444cb588fd25cb902c92_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,823 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111016d4cd2595c142c19093ed930897941a_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111016d4cd2595c142c19093ed930897941a_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111038d803a5ff594186aad86fca36893db6_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111038d803a5ff594186aad86fca36893db6_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,825 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039c80daaa6ee49d98c2a5b22cfbfe886_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111039c80daaa6ee49d98c2a5b22cfbfe886_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,826 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c238fbf5e3d456ca640ed5ca5220887_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c238fbf5e3d456ca640ed5ca5220887_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,827 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a5bb3af2fc0461a8e808448a19cbc1c_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106a5bb3af2fc0461a8e808448a19cbc1c_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,828 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107137a8ab98844c5aa075fe914ac563aa_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107137a8ab98844c5aa075fe914ac563aa_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,829 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108115e58caa024bc08eb378d1e69cae80_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108115e58caa024bc08eb378d1e69cae80_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,830 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110832380e1524b4629a570984115968839_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110832380e1524b4629a570984115968839_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,831 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108a4938655db4414b85b4d3c5cddd03ad_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108a4938655db4414b85b4d3c5cddd03ad_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,832 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110927820a3dee64a15a9e6be02d9c960d8_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110927820a3dee64a15a9e6be02d9c960d8_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,833 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110967fe2b69f45460b9251ec6004d48de8_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110967fe2b69f45460b9251ec6004d48de8_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,834 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110aee34dd0e5274dd8958319b82e191ca5_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110aee34dd0e5274dd8958319b82e191ca5_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,835 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2820e4faeed4d9bbc1d7a105b83259a_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2820e4faeed4d9bbc1d7a105b83259a_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,836 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c9f30a23cdab4a0b9e774544fcc3a8c3_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c9f30a23cdab4a0b9e774544fcc3a8c3_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,837 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d31a364300cc4bdd8435a699de7b8df9_5619dfa0d7c1c633ebe4b8aecdc69e58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d31a364300cc4bdd8435a699de7b8df9_5619dfa0d7c1c633ebe4b8aecdc69e58 2024-11-10T09:26:09,838 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:26:09,840 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,842 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:26:09,844 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:26:09,846 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,846 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:26:09,846 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230769846"}]},"ts":"9223372036854775807"} 2024-11-10T09:26:09,848 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:26:09,848 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5619dfa0d7c1c633ebe4b8aecdc69e58, NAME => 'TestAcidGuarantees,,1731230738946.5619dfa0d7c1c633ebe4b8aecdc69e58.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:26:09,848 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:26:09,849 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230769848"}]},"ts":"9223372036854775807"} 2024-11-10T09:26:09,851 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:26:09,852 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 76 msec 2024-11-10T09:26:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-10T09:26:09,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:26:09,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:09,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 238) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1008509712_22 at /127.0.0.1:38394 [Waiting for operation #577] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1990253602_22 at /127.0.0.1:55804 [Waiting for operation #498] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1008509712_22 at /127.0.0.1:55660 [Waiting for operation #559] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=459 (was 552), ProcessCount=11 (was 11), AvailableMemoryMB=6959 (was 7065) 2024-11-10T09:26:09,902 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=459, ProcessCount=11, AvailableMemoryMB=6959 2024-11-10T09:26:09,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:26:09,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:26:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:09,906 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:26:09,906 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:09,906 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-10T09:26:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T09:26:09,907 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:26:09,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742055_1231 (size=959) 2024-11-10T09:26:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T09:26:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T09:26:10,320 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:26:10,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742056_1232 (size=53) 2024-11-10T09:26:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T09:26:10,781 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:26:10,781 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 799270abaa9e80dc47bbc7cec6771c00, disabling compactions & flushes 2024-11-10T09:26:10,782 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:10,782 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:10,782 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. after waiting 0 ms 2024-11-10T09:26:10,782 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:10,782 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:10,782 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 799270abaa9e80dc47bbc7cec6771c00: Waiting for close lock at 1731230770781Disabling compacts and flushes for region at 1731230770781Disabling writes for close at 1731230770782 (+1 ms)Writing region close event to WAL at 1731230770782Closed at 1731230770782 2024-11-10T09:26:10,784 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:26:10,785 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230770784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230770784"}]},"ts":"1731230770784"} 2024-11-10T09:26:10,788 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:26:10,790 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:26:10,790 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230770790"}]},"ts":"1731230770790"} 2024-11-10T09:26:10,793 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:26:10,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, ASSIGN}] 2024-11-10T09:26:10,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, ASSIGN 2024-11-10T09:26:10,796 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, ASSIGN; state=OFFLINE, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=false 2024-11-10T09:26:10,947 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=799270abaa9e80dc47bbc7cec6771c00, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:10,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, ASSIGN because future has completed 2024-11-10T09:26:10,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:26:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T09:26:11,110 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:11,110 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:26:11,111 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,111 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:26:11,111 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,111 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,113 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,115 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 799270abaa9e80dc47bbc7cec6771c00 columnFamilyName A 2024-11-10T09:26:11,115 DEBUG [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:11,116 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] regionserver.HStore(327): Store=799270abaa9e80dc47bbc7cec6771c00/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:11,116 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,119 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 799270abaa9e80dc47bbc7cec6771c00 columnFamilyName B 2024-11-10T09:26:11,119 DEBUG [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:11,120 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] regionserver.HStore(327): Store=799270abaa9e80dc47bbc7cec6771c00/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:11,120 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,122 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 799270abaa9e80dc47bbc7cec6771c00 columnFamilyName C 2024-11-10T09:26:11,122 DEBUG [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:11,123 INFO [StoreOpener-799270abaa9e80dc47bbc7cec6771c00-1 {}] regionserver.HStore(327): Store=799270abaa9e80dc47bbc7cec6771c00/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:11,123 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,125 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,125 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,127 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,127 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,128 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:26:11,131 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,134 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:26:11,134 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 799270abaa9e80dc47bbc7cec6771c00; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62804185, jitterRate=-0.06414471566677094}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:26:11,134 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:11,135 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 799270abaa9e80dc47bbc7cec6771c00: Running coprocessor pre-open hook at 1731230771111Writing region info on filesystem at 1731230771111Initializing all the Stores at 1731230771112 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230771112Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230771113 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230771113Cleaning up temporary data from old regions at 1731230771127 (+14 ms)Running coprocessor post-open hooks at 1731230771134 (+7 ms)Region opened successfully at 1731230771135 (+1 ms) 2024-11-10T09:26:11,136 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., pid=65, masterSystemTime=1731230771106 2024-11-10T09:26:11,138 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:11,138 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:11,139 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=799270abaa9e80dc47bbc7cec6771c00, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:11,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:26:11,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T09:26:11,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 in 190 msec 2024-11-10T09:26:11,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-10T09:26:11,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, ASSIGN in 349 msec 2024-11-10T09:26:11,147 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:26:11,147 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230771147"}]},"ts":"1731230771147"} 2024-11-10T09:26:11,149 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:26:11,150 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:26:11,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2460 sec 2024-11-10T09:26:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-10T09:26:12,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:12,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce28163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,070 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,070 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,072 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226af071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,075 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169c3314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,081 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:26:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,38235,1731230703274 2024-11-10T09:26:12,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@241d4000 2024-11-10T09:26:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:26:12,084 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:26:12,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7642a08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,086 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eec9ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,088 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28011a7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecfb794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,090 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,090 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51678a7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,092 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c35c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0f448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,095 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7083994a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,096 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4a2691, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d560643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,099 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563a84eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2941138d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6509bdf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,103 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19cdaa5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,104 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e30c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@beb69b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,106 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b9d7cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@80a1371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643da6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,109 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eca4068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,110 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c67b9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829bcb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,112 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b25938e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,114 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c27504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d96952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,115 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ecf7de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,117 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee4223d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24595b5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:12,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:12,118 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2cf695, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:12,119 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d5687f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:12,120 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,120 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,120 DEBUG [Thread-1096 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,120 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,121 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,121 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,121 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,121 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,121 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,122 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T09:26:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,123 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:26:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,124 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:12,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:12,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:12,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:12,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:26:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:12,137 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:12,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:26:12,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/cb1e5ce642864c56951fc08ad8ebd181 is 50, key is test_row_0/A:col10/1731230772136/Put/seqid=0 2024-11-10T09:26:12,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742057_1233 (size=12001) 2024-11-10T09:26:12,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/cb1e5ce642864c56951fc08ad8ebd181 2024-11-10T09:26:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230832169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230832172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230832174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230832177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230832177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/fd056bb70eb9405988bc3e4f3b464d39 is 50, key is test_row_0/B:col10/1731230772136/Put/seqid=0 2024-11-10T09:26:12,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742058_1234 (size=12001) 2024-11-10T09:26:12,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/fd056bb70eb9405988bc3e4f3b464d39 2024-11-10T09:26:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:26:12,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/f1bc68c6ade849aa83a5bf99dac8d8c8 is 50, key is test_row_0/C:col10/1731230772136/Put/seqid=0 2024-11-10T09:26:12,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742059_1235 (size=12001) 2024-11-10T09:26:12,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/f1bc68c6ade849aa83a5bf99dac8d8c8 2024-11-10T09:26:12,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/cb1e5ce642864c56951fc08ad8ebd181 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/cb1e5ce642864c56951fc08ad8ebd181 2024-11-10T09:26:12,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T09:26:12,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:12,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:12,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:12,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:12,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/cb1e5ce642864c56951fc08ad8ebd181, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:26:12,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/fd056bb70eb9405988bc3e4f3b464d39 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd056bb70eb9405988bc3e4f3b464d39 2024-11-10T09:26:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230832286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230832286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230832286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230832287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd056bb70eb9405988bc3e4f3b464d39, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:26:12,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/f1bc68c6ade849aa83a5bf99dac8d8c8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f1bc68c6ade849aa83a5bf99dac8d8c8 2024-11-10T09:26:12,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f1bc68c6ade849aa83a5bf99dac8d8c8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T09:26:12,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230832296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 799270abaa9e80dc47bbc7cec6771c00 in 165ms, sequenceid=13, compaction requested=false 2024-11-10T09:26:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T09:26:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:12,433 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:26:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:26:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9 is 50, key is test_row_0/A:col10/1731230772173/Put/seqid=0 2024-11-10T09:26:12,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742060_1236 (size=12001) 2024-11-10T09:26:12,467 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9 2024-11-10T09:26:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3e2b26eb65004997bef4dc12c277bc8b is 50, key is test_row_0/B:col10/1731230772173/Put/seqid=0 2024-11-10T09:26:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742061_1237 (size=12001) 2024-11-10T09:26:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:12,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230832504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230832504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230832505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230832505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230832506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230832616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230832616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230832616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230832616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:26:12,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230832816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230832825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230832825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230832825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230832825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:12,885 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3e2b26eb65004997bef4dc12c277bc8b 2024-11-10T09:26:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/9d10e6995b344fb299f4bffa79c8c624 is 50, key is test_row_0/C:col10/1731230772173/Put/seqid=0 2024-11-10T09:26:12,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742062_1238 (size=12001) 2024-11-10T09:26:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230833136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230833136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230833136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230833136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:26:13,316 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/9d10e6995b344fb299f4bffa79c8c624 2024-11-10T09:26:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9 2024-11-10T09:26:13,324 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T09:26:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3e2b26eb65004997bef4dc12c277bc8b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3e2b26eb65004997bef4dc12c277bc8b 2024-11-10T09:26:13,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230833326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,328 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3e2b26eb65004997bef4dc12c277bc8b, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T09:26:13,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/9d10e6995b344fb299f4bffa79c8c624 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9d10e6995b344fb299f4bffa79c8c624 2024-11-10T09:26:13,333 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9d10e6995b344fb299f4bffa79c8c624, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T09:26:13,333 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 799270abaa9e80dc47bbc7cec6771c00 in 900ms, sequenceid=38, compaction requested=false 2024-11-10T09:26:13,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:13,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:13,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T09:26:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T09:26:13,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T09:26:13,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2100 sec 2024-11-10T09:26:13,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.2160 sec 2024-11-10T09:26:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:13,647 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:13,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:26:13,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/9bc934b949544d56981af7da4fe083fe is 50, key is test_row_0/A:col10/1731230773647/Put/seqid=0 2024-11-10T09:26:13,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742063_1239 (size=16681) 2024-11-10T09:26:13,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/9bc934b949544d56981af7da4fe083fe 2024-11-10T09:26:13,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230833676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230833677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/c99d2234870242e9a1172d637cdd778a is 50, key is test_row_0/B:col10/1731230773647/Put/seqid=0 2024-11-10T09:26:13,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230833683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230833684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742064_1240 (size=12001) 2024-11-10T09:26:13,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/c99d2234870242e9a1172d637cdd778a 2024-11-10T09:26:13,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/bf91afc4e9e2461db44eb8106b8c27b6 is 50, key is test_row_0/C:col10/1731230773647/Put/seqid=0 2024-11-10T09:26:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742065_1241 (size=12001) 2024-11-10T09:26:13,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/bf91afc4e9e2461db44eb8106b8c27b6 2024-11-10T09:26:13,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/9bc934b949544d56981af7da4fe083fe as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9bc934b949544d56981af7da4fe083fe 2024-11-10T09:26:13,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9bc934b949544d56981af7da4fe083fe, entries=250, sequenceid=51, filesize=16.3 K 2024-11-10T09:26:13,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/c99d2234870242e9a1172d637cdd778a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/c99d2234870242e9a1172d637cdd778a 2024-11-10T09:26:13,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/c99d2234870242e9a1172d637cdd778a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T09:26:13,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/bf91afc4e9e2461db44eb8106b8c27b6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/bf91afc4e9e2461db44eb8106b8c27b6 2024-11-10T09:26:13,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/bf91afc4e9e2461db44eb8106b8c27b6, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T09:26:13,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 799270abaa9e80dc47bbc7cec6771c00 in 89ms, sequenceid=51, compaction requested=true 2024-11-10T09:26:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:13,736 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:13,736 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:13,738 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:13,738 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:13,738 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/A is initiating minor compaction (all files) 2024-11-10T09:26:13,738 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/B is initiating minor compaction (all files) 2024-11-10T09:26:13,738 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/A in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:13,738 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/B in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:13,738 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd056bb70eb9405988bc3e4f3b464d39, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3e2b26eb65004997bef4dc12c277bc8b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/c99d2234870242e9a1172d637cdd778a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=35.2 K 2024-11-10T09:26:13,738 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/cb1e5ce642864c56951fc08ad8ebd181, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9bc934b949544d56981af7da4fe083fe] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=39.7 K 2024-11-10T09:26:13,739 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1e5ce642864c56951fc08ad8ebd181, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230772130 2024-11-10T09:26:13,739 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting fd056bb70eb9405988bc3e4f3b464d39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230772130 2024-11-10T09:26:13,739 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0edf2e2adcb94ba08cd9a2fdc2bfcdf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731230772172 2024-11-10T09:26:13,739 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e2b26eb65004997bef4dc12c277bc8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731230772172 2024-11-10T09:26:13,739 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bc934b949544d56981af7da4fe083fe, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230772498 2024-11-10T09:26:13,739 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c99d2234870242e9a1172d637cdd778a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230772502 2024-11-10T09:26:13,750 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#A#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:13,751 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/ad42b06c741a4990ac9eaf759f28d88e is 50, key is test_row_0/A:col10/1731230773647/Put/seqid=0 2024-11-10T09:26:13,753 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:13,754 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/7802943694af4530b50ee8fa51540806 is 50, key is test_row_0/B:col10/1731230773647/Put/seqid=0 2024-11-10T09:26:13,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742067_1243 (size=12104) 2024-11-10T09:26:13,783 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/7802943694af4530b50ee8fa51540806 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/7802943694af4530b50ee8fa51540806 2024-11-10T09:26:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:13,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:13,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:26:13,789 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/B of 799270abaa9e80dc47bbc7cec6771c00 into 7802943694af4530b50ee8fa51540806(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:13,789 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:13,789 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/B, priority=13, startTime=1731230773736; duration=0sec 2024-11-10T09:26:13,789 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:13,789 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:B 2024-11-10T09:26:13,789 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:13,791 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:13,792 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/C is initiating minor compaction (all files) 2024-11-10T09:26:13,792 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/C in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:13,792 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f1bc68c6ade849aa83a5bf99dac8d8c8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9d10e6995b344fb299f4bffa79c8c624, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/bf91afc4e9e2461db44eb8106b8c27b6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=35.2 K 2024-11-10T09:26:13,793 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f1bc68c6ade849aa83a5bf99dac8d8c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731230772130 2024-11-10T09:26:13,793 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d10e6995b344fb299f4bffa79c8c624, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731230772172 2024-11-10T09:26:13,794 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting bf91afc4e9e2461db44eb8106b8c27b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230772502 2024-11-10T09:26:13,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c6793d11ebc24dd5971bad35097122fd is 50, key is test_row_0/A:col10/1731230773673/Put/seqid=0 2024-11-10T09:26:13,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742066_1242 (size=12104) 2024-11-10T09:26:13,803 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:13,803 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/22a7f46e282348ff91e8a721d95788fc is 50, key is test_row_0/C:col10/1731230773647/Put/seqid=0 2024-11-10T09:26:13,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742068_1244 (size=14341) 2024-11-10T09:26:13,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230833805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230833805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230833808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742069_1245 (size=12104) 2024-11-10T09:26:13,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230833808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230833916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230833916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230833916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230833916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230834126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230834126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230834126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230834126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,206 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/ad42b06c741a4990ac9eaf759f28d88e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ad42b06c741a4990ac9eaf759f28d88e 2024-11-10T09:26:14,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c6793d11ebc24dd5971bad35097122fd 2024-11-10T09:26:14,210 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/A of 799270abaa9e80dc47bbc7cec6771c00 into ad42b06c741a4990ac9eaf759f28d88e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:14,210 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:14,210 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/A, priority=13, startTime=1731230773736; duration=0sec 2024-11-10T09:26:14,211 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:14,211 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:A 2024-11-10T09:26:14,217 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/22a7f46e282348ff91e8a721d95788fc as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/22a7f46e282348ff91e8a721d95788fc 2024-11-10T09:26:14,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/687c68e9061f48469842a5156aeb2495 is 50, key is test_row_0/B:col10/1731230773673/Put/seqid=0 2024-11-10T09:26:14,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742070_1246 (size=12001) 2024-11-10T09:26:14,226 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/C of 799270abaa9e80dc47bbc7cec6771c00 into 22a7f46e282348ff91e8a721d95788fc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:14,226 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:14,226 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/C, priority=13, startTime=1731230773736; duration=0sec 2024-11-10T09:26:14,226 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:14,227 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:C 2024-11-10T09:26:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T09:26:14,266 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:14,266 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T09:26:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:26:14,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:14,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:14,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:14,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230834336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,338 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 799270abaa9e80dc47bbc7cec6771c00 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:26:14,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:26:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230834436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230834436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230834436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230834436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:26:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:26:14,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/687c68e9061f48469842a5156aeb2495 2024-11-10T09:26:14,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/dad5915db1ee497aadc7a6dde1a5a502 is 50, key is test_row_0/C:col10/1731230773673/Put/seqid=0 2024-11-10T09:26:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742071_1247 (size=12001) 2024-11-10T09:26:14,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:26:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:26:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:14,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:26:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230834945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230834945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230834945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230834945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:15,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:26:15,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:15,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:15,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:15,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:15,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:15,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/dad5915db1ee497aadc7a6dde1a5a502 2024-11-10T09:26:15,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c6793d11ebc24dd5971bad35097122fd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c6793d11ebc24dd5971bad35097122fd 2024-11-10T09:26:15,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c6793d11ebc24dd5971bad35097122fd, entries=200, sequenceid=78, filesize=14.0 K 2024-11-10T09:26:15,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/687c68e9061f48469842a5156aeb2495 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/687c68e9061f48469842a5156aeb2495 2024-11-10T09:26:15,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/687c68e9061f48469842a5156aeb2495, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T09:26:15,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/dad5915db1ee497aadc7a6dde1a5a502 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dad5915db1ee497aadc7a6dde1a5a502 2024-11-10T09:26:15,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dad5915db1ee497aadc7a6dde1a5a502, entries=150, sequenceid=78, filesize=11.7 K 2024-11-10T09:26:15,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 799270abaa9e80dc47bbc7cec6771c00 in 1265ms, sequenceid=78, compaction requested=false 2024-11-10T09:26:15,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:15,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T09:26:15,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:15,185 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T09:26:15,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/64201ba9cbe74d2f8a3f20673ee3ea1d is 50, key is test_row_0/A:col10/1731230773807/Put/seqid=0 2024-11-10T09:26:15,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742072_1248 (size=12001) 2024-11-10T09:26:15,198 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/64201ba9cbe74d2f8a3f20673ee3ea1d 2024-11-10T09:26:15,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/9f5dbfe8b28f43c8aeb07882772a0d72 is 50, key is test_row_0/B:col10/1731230773807/Put/seqid=0 2024-11-10T09:26:15,209 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:26:15,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742073_1249 (size=12001) 2024-11-10T09:26:15,226 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/9f5dbfe8b28f43c8aeb07882772a0d72 2024-11-10T09:26:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/6cdeeabd39f84b5782fe6dbb8ab2705c is 50, key is test_row_0/C:col10/1731230773807/Put/seqid=0 2024-11-10T09:26:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742074_1250 (size=12001) 2024-11-10T09:26:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:26:15,662 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/6cdeeabd39f84b5782fe6dbb8ab2705c 2024-11-10T09:26:15,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/64201ba9cbe74d2f8a3f20673ee3ea1d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/64201ba9cbe74d2f8a3f20673ee3ea1d 2024-11-10T09:26:15,672 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/64201ba9cbe74d2f8a3f20673ee3ea1d, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T09:26:15,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/9f5dbfe8b28f43c8aeb07882772a0d72 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/9f5dbfe8b28f43c8aeb07882772a0d72 2024-11-10T09:26:15,677 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/9f5dbfe8b28f43c8aeb07882772a0d72, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T09:26:15,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/6cdeeabd39f84b5782fe6dbb8ab2705c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/6cdeeabd39f84b5782fe6dbb8ab2705c 2024-11-10T09:26:15,683 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/6cdeeabd39f84b5782fe6dbb8ab2705c, entries=150, sequenceid=90, filesize=11.7 K 2024-11-10T09:26:15,684 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 799270abaa9e80dc47bbc7cec6771c00 in 499ms, sequenceid=90, compaction requested=true 2024-11-10T09:26:15,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:15,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:15,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T09:26:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T09:26:15,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T09:26:15,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-10T09:26:15,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4220 sec 2024-11-10T09:26:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:15,962 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:15,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:26:15,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/4d3bc033bec34a7b90535e283fb1f4c9 is 50, key is test_row_0/A:col10/1731230775960/Put/seqid=0 2024-11-10T09:26:15,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742075_1251 (size=12001) 2024-11-10T09:26:15,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/4d3bc033bec34a7b90535e283fb1f4c9 2024-11-10T09:26:15,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/f3a66fecd4d84479b47fa8a866b366fc is 50, key is test_row_0/B:col10/1731230775960/Put/seqid=0 2024-11-10T09:26:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230835990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230835990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:15,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230835995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230835996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742076_1252 (size=12001) 2024-11-10T09:26:16,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/f3a66fecd4d84479b47fa8a866b366fc 2024-11-10T09:26:16,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/2cf4e64ca1ab46ae838e2876c0719f50 is 50, key is test_row_0/C:col10/1731230775960/Put/seqid=0 2024-11-10T09:26:16,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742077_1253 (size=12001) 2024-11-10T09:26:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230836105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230836106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230836106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230836106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230836315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230836315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230836316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230836316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230836345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,347 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 799270abaa9e80dc47bbc7cec6771c00 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/2cf4e64ca1ab46ae838e2876c0719f50 2024-11-10T09:26:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T09:26:16,416 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:16,416 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:16,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/4d3bc033bec34a7b90535e283fb1f4c9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/4d3bc033bec34a7b90535e283fb1f4c9 2024-11-10T09:26:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T09:26:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:26:16,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:16,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:16,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:16,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/4d3bc033bec34a7b90535e283fb1f4c9, entries=150, sequenceid=101, filesize=11.7 K 2024-11-10T09:26:16,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/f3a66fecd4d84479b47fa8a866b366fc as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f3a66fecd4d84479b47fa8a866b366fc 2024-11-10T09:26:16,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f3a66fecd4d84479b47fa8a866b366fc, entries=150, sequenceid=101, filesize=11.7 K 2024-11-10T09:26:16,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/2cf4e64ca1ab46ae838e2876c0719f50 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/2cf4e64ca1ab46ae838e2876c0719f50 2024-11-10T09:26:16,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/2cf4e64ca1ab46ae838e2876c0719f50, entries=150, sequenceid=101, filesize=11.7 K 2024-11-10T09:26:16,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 799270abaa9e80dc47bbc7cec6771c00 in 470ms, sequenceid=101, compaction requested=true 2024-11-10T09:26:16,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:16,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:16,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:16,432 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:26:16,432 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:26:16,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:16,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:16,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:16,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:16,433 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:26:16,433 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:26:16,433 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/B is initiating minor compaction (all files) 2024-11-10T09:26:16,433 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/A is initiating minor compaction (all files) 2024-11-10T09:26:16,433 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/B in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:16,433 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/A in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:16,434 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/7802943694af4530b50ee8fa51540806, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/687c68e9061f48469842a5156aeb2495, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/9f5dbfe8b28f43c8aeb07882772a0d72, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f3a66fecd4d84479b47fa8a866b366fc] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=47.0 K 2024-11-10T09:26:16,434 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ad42b06c741a4990ac9eaf759f28d88e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c6793d11ebc24dd5971bad35097122fd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/64201ba9cbe74d2f8a3f20673ee3ea1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/4d3bc033bec34a7b90535e283fb1f4c9] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=49.3 K 2024-11-10T09:26:16,434 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7802943694af4530b50ee8fa51540806, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230772502 2024-11-10T09:26:16,434 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad42b06c741a4990ac9eaf759f28d88e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230772502 2024-11-10T09:26:16,434 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6793d11ebc24dd5971bad35097122fd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731230773673 2024-11-10T09:26:16,434 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 687c68e9061f48469842a5156aeb2495, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731230773673 2024-11-10T09:26:16,434 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64201ba9cbe74d2f8a3f20673ee3ea1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230773798 2024-11-10T09:26:16,434 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5dbfe8b28f43c8aeb07882772a0d72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230773798 2024-11-10T09:26:16,435 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3bc033bec34a7b90535e283fb1f4c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731230775960 2024-11-10T09:26:16,435 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a66fecd4d84479b47fa8a866b366fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731230775960 2024-11-10T09:26:16,443 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#B#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:16,444 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:16,444 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/b507aa37518b42c5a67ec3888821fd43 is 50, key is test_row_0/B:col10/1731230775960/Put/seqid=0 2024-11-10T09:26:16,444 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/dfd89a8926814252bc023214946b3771 is 50, key is test_row_0/A:col10/1731230775960/Put/seqid=0 2024-11-10T09:26:16,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742078_1254 (size=12241) 2024-11-10T09:26:16,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742079_1255 (size=12241) 2024-11-10T09:26:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:26:16,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T09:26:16,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:16,571 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:26:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/6b24ae31451440f2917032df53fc811a is 50, key is test_row_0/A:col10/1731230775987/Put/seqid=0 2024-11-10T09:26:16,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742080_1256 (size=12001) 2024-11-10T09:26:16,584 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/6b24ae31451440f2917032df53fc811a 2024-11-10T09:26:16,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/320e025ad0a94d26a4e6a1ebb6d4288d is 50, key is test_row_0/B:col10/1731230775987/Put/seqid=0 2024-11-10T09:26:16,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742081_1257 (size=12001) 2024-11-10T09:26:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:16,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:16,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230836632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230836634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230836636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230836637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:26:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230836746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230836746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230836746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230836746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,855 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/b507aa37518b42c5a67ec3888821fd43 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/b507aa37518b42c5a67ec3888821fd43 2024-11-10T09:26:16,860 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/B of 799270abaa9e80dc47bbc7cec6771c00 into b507aa37518b42c5a67ec3888821fd43(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:16,860 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:16,860 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/B, priority=12, startTime=1731230776432; duration=0sec 2024-11-10T09:26:16,861 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:16,861 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:B 2024-11-10T09:26:16,861 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:26:16,862 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:26:16,862 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/C is initiating minor compaction (all files) 2024-11-10T09:26:16,862 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/C in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:16,862 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/22a7f46e282348ff91e8a721d95788fc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dad5915db1ee497aadc7a6dde1a5a502, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/6cdeeabd39f84b5782fe6dbb8ab2705c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/2cf4e64ca1ab46ae838e2876c0719f50] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=47.0 K 2024-11-10T09:26:16,863 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a7f46e282348ff91e8a721d95788fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731230772502 2024-11-10T09:26:16,863 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting dad5915db1ee497aadc7a6dde1a5a502, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731230773673 2024-11-10T09:26:16,863 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdeeabd39f84b5782fe6dbb8ab2705c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230773798 2024-11-10T09:26:16,864 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cf4e64ca1ab46ae838e2876c0719f50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731230775960 2024-11-10T09:26:16,865 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/dfd89a8926814252bc023214946b3771 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/dfd89a8926814252bc023214946b3771 2024-11-10T09:26:16,870 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/A of 799270abaa9e80dc47bbc7cec6771c00 into dfd89a8926814252bc023214946b3771(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:16,870 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:16,870 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/A, priority=12, startTime=1731230776432; duration=0sec 2024-11-10T09:26:16,870 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:16,870 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:A 2024-11-10T09:26:16,873 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:16,873 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/58407eab4f3541b3851345e88508ce1d is 50, key is test_row_0/C:col10/1731230775960/Put/seqid=0 2024-11-10T09:26:16,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742082_1258 (size=12241) 2024-11-10T09:26:16,882 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/58407eab4f3541b3851345e88508ce1d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58407eab4f3541b3851345e88508ce1d 2024-11-10T09:26:16,886 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/C of 799270abaa9e80dc47bbc7cec6771c00 into 58407eab4f3541b3851345e88508ce1d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:16,886 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:16,886 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/C, priority=12, startTime=1731230776432; duration=0sec 2024-11-10T09:26:16,886 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:16,886 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:C 2024-11-10T09:26:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230836956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230836956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230836956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230836956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:16,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:16,997 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/320e025ad0a94d26a4e6a1ebb6d4288d 2024-11-10T09:26:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/11316933df674726b026d870473d3151 is 50, key is test_row_0/C:col10/1731230775987/Put/seqid=0 2024-11-10T09:26:17,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742083_1259 (size=12001) 2024-11-10T09:26:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:26:17,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230837266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230837266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230837266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230837266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,417 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/11316933df674726b026d870473d3151 2024-11-10T09:26:17,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/6b24ae31451440f2917032df53fc811a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6b24ae31451440f2917032df53fc811a 2024-11-10T09:26:17,425 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6b24ae31451440f2917032df53fc811a, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T09:26:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/320e025ad0a94d26a4e6a1ebb6d4288d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/320e025ad0a94d26a4e6a1ebb6d4288d 2024-11-10T09:26:17,429 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/320e025ad0a94d26a4e6a1ebb6d4288d, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T09:26:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/11316933df674726b026d870473d3151 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/11316933df674726b026d870473d3151 2024-11-10T09:26:17,435 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/11316933df674726b026d870473d3151, entries=150, sequenceid=126, filesize=11.7 K 2024-11-10T09:26:17,435 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 799270abaa9e80dc47bbc7cec6771c00 in 864ms, sequenceid=126, compaction requested=false 2024-11-10T09:26:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T09:26:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T09:26:17,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T09:26:17,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-11-10T09:26:17,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0230 sec 2024-11-10T09:26:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T09:26:17,555 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:17,556 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T09:26:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:26:17,557 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:17,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:17,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:26:17,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T09:26:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:17,710 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:26:17,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/060dd9dee2614c688b8b49f837cd5ac5 is 50, key is test_row_0/A:col10/1731230776636/Put/seqid=0 2024-11-10T09:26:17,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742084_1260 (size=12151) 2024-11-10T09:26:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:17,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230837800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230837801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230837802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230837803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:26:17,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230837916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230837916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230837916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:17,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230837916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230838125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230838125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230838126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230838126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,131 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/060dd9dee2614c688b8b49f837cd5ac5 2024-11-10T09:26:18,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/ace483bb41fd4a8281f4c611780e4ebd is 50, key is test_row_0/B:col10/1731230776636/Put/seqid=0 2024-11-10T09:26:18,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742085_1261 (size=12151) 2024-11-10T09:26:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:26:18,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230838435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230838435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230838436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230838436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,542 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/ace483bb41fd4a8281f4c611780e4ebd 2024-11-10T09:26:18,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/14a4e5413b664ee2a6efdb8901a08565 is 50, key is test_row_0/C:col10/1731230776636/Put/seqid=0 2024-11-10T09:26:18,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742086_1262 (size=12151) 2024-11-10T09:26:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:26:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230838945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230838946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230838946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230838946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:18,954 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/14a4e5413b664ee2a6efdb8901a08565 2024-11-10T09:26:18,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/060dd9dee2614c688b8b49f837cd5ac5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/060dd9dee2614c688b8b49f837cd5ac5 2024-11-10T09:26:18,962 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/060dd9dee2614c688b8b49f837cd5ac5, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T09:26:18,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/ace483bb41fd4a8281f4c611780e4ebd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ace483bb41fd4a8281f4c611780e4ebd 2024-11-10T09:26:18,967 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ace483bb41fd4a8281f4c611780e4ebd, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T09:26:18,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/14a4e5413b664ee2a6efdb8901a08565 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/14a4e5413b664ee2a6efdb8901a08565 2024-11-10T09:26:18,971 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/14a4e5413b664ee2a6efdb8901a08565, entries=150, sequenceid=140, filesize=11.9 K 2024-11-10T09:26:18,972 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 799270abaa9e80dc47bbc7cec6771c00 in 1262ms, sequenceid=140, compaction requested=true 2024-11-10T09:26:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:18,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T09:26:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T09:26:18,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T09:26:18,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-10T09:26:18,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4200 sec 2024-11-10T09:26:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T09:26:19,705 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:19,706 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T09:26:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:26:19,707 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:19,708 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:19,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:26:19,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T09:26:19,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:19,860 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:26:19,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/e1d702dd695341ec885b409890595c58 is 50, key is test_row_0/A:col10/1731230777801/Put/seqid=0 2024-11-10T09:26:19,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742087_1263 (size=12151) 2024-11-10T09:26:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:19,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230839967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230839970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230839970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:19,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230839971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:19,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:26:20,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230840076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230840086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,268 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/e1d702dd695341ec885b409890595c58 2024-11-10T09:26:20,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/532ebb8107a54267b3bba6f6412a2070 is 50, key is test_row_0/B:col10/1731230777801/Put/seqid=0 2024-11-10T09:26:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742088_1264 (size=12151) 2024-11-10T09:26:20,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230840286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230840296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:26:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230840376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,379 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 799270abaa9e80dc47bbc7cec6771c00 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230840596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230840606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:20,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:20,679 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/532ebb8107a54267b3bba6f6412a2070 2024-11-10T09:26:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/a192dadc7898415eb77b4fc133941f04 is 50, key is test_row_0/C:col10/1731230777801/Put/seqid=0 2024-11-10T09:26:20,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742089_1265 (size=12151) 2024-11-10T09:26:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:26:21,090 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/a192dadc7898415eb77b4fc133941f04 2024-11-10T09:26:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/e1d702dd695341ec885b409890595c58 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/e1d702dd695341ec885b409890595c58 2024-11-10T09:26:21,098 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/e1d702dd695341ec885b409890595c58, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T09:26:21,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/532ebb8107a54267b3bba6f6412a2070 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/532ebb8107a54267b3bba6f6412a2070 2024-11-10T09:26:21,102 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/532ebb8107a54267b3bba6f6412a2070, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T09:26:21,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/a192dadc7898415eb77b4fc133941f04 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a192dadc7898415eb77b4fc133941f04 2024-11-10T09:26:21,106 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a192dadc7898415eb77b4fc133941f04, entries=150, sequenceid=165, filesize=11.9 K 2024-11-10T09:26:21,107 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 799270abaa9e80dc47bbc7cec6771c00 in 1247ms, sequenceid=165, compaction requested=true 2024-11-10T09:26:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T09:26:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T09:26:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:21,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:21,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:26:21,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T09:26:21,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-10T09:26:21,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4040 sec 2024-11-10T09:26:21,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/1e3eee8ab1804a75818699af31a3f09c is 50, key is test_row_0/A:col10/1731230779969/Put/seqid=0 2024-11-10T09:26:21,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742090_1266 (size=14541) 2024-11-10T09:26:21,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/1e3eee8ab1804a75818699af31a3f09c 2024-11-10T09:26:21,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/d2852939575044059944114cdd007f74 is 50, key is test_row_0/B:col10/1731230779969/Put/seqid=0 2024-11-10T09:26:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742091_1267 (size=12151) 2024-11-10T09:26:21,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/d2852939575044059944114cdd007f74 2024-11-10T09:26:21,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/7f1c5655e4b145bd859cedaac281fae3 is 50, key is test_row_0/C:col10/1731230779969/Put/seqid=0 2024-11-10T09:26:21,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742092_1268 (size=12151) 2024-11-10T09:26:21,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230841144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230841148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230841148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230841149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230841255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230841266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230841266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230841266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230841465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230841476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230841476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230841476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/7f1c5655e4b145bd859cedaac281fae3 2024-11-10T09:26:21,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/1e3eee8ab1804a75818699af31a3f09c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/1e3eee8ab1804a75818699af31a3f09c 2024-11-10T09:26:21,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/1e3eee8ab1804a75818699af31a3f09c, entries=200, sequenceid=178, filesize=14.2 K 2024-11-10T09:26:21,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/d2852939575044059944114cdd007f74 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d2852939575044059944114cdd007f74 2024-11-10T09:26:21,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d2852939575044059944114cdd007f74, entries=150, sequenceid=178, filesize=11.9 K 2024-11-10T09:26:21,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/7f1c5655e4b145bd859cedaac281fae3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/7f1c5655e4b145bd859cedaac281fae3 2024-11-10T09:26:21,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/7f1c5655e4b145bd859cedaac281fae3, entries=150, sequenceid=178, filesize=11.9 K 2024-11-10T09:26:21,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 799270abaa9e80dc47bbc7cec6771c00 in 454ms, sequenceid=178, compaction requested=true 2024-11-10T09:26:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:21,563 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:26:21,563 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:26:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:21,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:21,564 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63085 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:26:21,565 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60695 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:26:21,565 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/A is initiating minor compaction (all files) 2024-11-10T09:26:21,565 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/B is initiating minor compaction (all files) 2024-11-10T09:26:21,565 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/A in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:21,565 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/B in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:21,565 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/dfd89a8926814252bc023214946b3771, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6b24ae31451440f2917032df53fc811a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/060dd9dee2614c688b8b49f837cd5ac5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/e1d702dd695341ec885b409890595c58, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/1e3eee8ab1804a75818699af31a3f09c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=61.6 K 2024-11-10T09:26:21,565 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/b507aa37518b42c5a67ec3888821fd43, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/320e025ad0a94d26a4e6a1ebb6d4288d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ace483bb41fd4a8281f4c611780e4ebd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/532ebb8107a54267b3bba6f6412a2070, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d2852939575044059944114cdd007f74] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=59.3 K 2024-11-10T09:26:21,565 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfd89a8926814252bc023214946b3771, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731230775960 2024-11-10T09:26:21,565 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting b507aa37518b42c5a67ec3888821fd43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731230775960 2024-11-10T09:26:21,566 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 320e025ad0a94d26a4e6a1ebb6d4288d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731230775987 2024-11-10T09:26:21,566 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b24ae31451440f2917032df53fc811a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731230775987 2024-11-10T09:26:21,566 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ace483bb41fd4a8281f4c611780e4ebd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731230776631 2024-11-10T09:26:21,566 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 060dd9dee2614c688b8b49f837cd5ac5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731230776631 2024-11-10T09:26:21,566 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 532ebb8107a54267b3bba6f6412a2070, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731230777800 2024-11-10T09:26:21,566 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1d702dd695341ec885b409890595c58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731230777800 2024-11-10T09:26:21,566 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d2852939575044059944114cdd007f74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731230779968 2024-11-10T09:26:21,567 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e3eee8ab1804a75818699af31a3f09c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731230779966 2024-11-10T09:26:21,580 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#B#compaction#227 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:21,580 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:21,581 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/480ce0ad444d416886328b87f22f2ea4 is 50, key is test_row_0/B:col10/1731230779969/Put/seqid=0 2024-11-10T09:26:21,581 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/fb5c1ec2813a47ea866e579497973564 is 50, key is test_row_0/A:col10/1731230779969/Put/seqid=0 2024-11-10T09:26:21,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742093_1269 (size=12561) 2024-11-10T09:26:21,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742094_1270 (size=12561) 2024-11-10T09:26:21,593 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/480ce0ad444d416886328b87f22f2ea4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/480ce0ad444d416886328b87f22f2ea4 2024-11-10T09:26:21,593 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/fb5c1ec2813a47ea866e579497973564 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/fb5c1ec2813a47ea866e579497973564 2024-11-10T09:26:21,599 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/A of 799270abaa9e80dc47bbc7cec6771c00 into fb5c1ec2813a47ea866e579497973564(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:21,599 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:21,599 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/B of 799270abaa9e80dc47bbc7cec6771c00 into 480ce0ad444d416886328b87f22f2ea4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:21,599 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/A, priority=11, startTime=1731230781563; duration=0sec 2024-11-10T09:26:21,599 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:21,599 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/B, priority=11, startTime=1731230781563; duration=0sec 2024-11-10T09:26:21,599 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:21,599 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:21,599 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:A 2024-11-10T09:26:21,599 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:B 2024-11-10T09:26:21,599 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:26:21,601 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60695 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:26:21,601 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/C is initiating minor compaction (all files) 2024-11-10T09:26:21,601 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/C in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:21,601 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58407eab4f3541b3851345e88508ce1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/11316933df674726b026d870473d3151, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/14a4e5413b664ee2a6efdb8901a08565, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a192dadc7898415eb77b4fc133941f04, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/7f1c5655e4b145bd859cedaac281fae3] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=59.3 K 2024-11-10T09:26:21,601 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58407eab4f3541b3851345e88508ce1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731230775960 2024-11-10T09:26:21,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11316933df674726b026d870473d3151, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731230775987 2024-11-10T09:26:21,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a4e5413b664ee2a6efdb8901a08565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731230776631 2024-11-10T09:26:21,602 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting a192dadc7898415eb77b4fc133941f04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731230777800 2024-11-10T09:26:21,603 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f1c5655e4b145bd859cedaac281fae3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731230779968 2024-11-10T09:26:21,615 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#C#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:21,615 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/9c0539e123c1495d8d1d930a046895c7 is 50, key is test_row_0/C:col10/1731230779969/Put/seqid=0 2024-11-10T09:26:21,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742095_1271 (size=12561) 2024-11-10T09:26:21,642 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/9c0539e123c1495d8d1d930a046895c7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9c0539e123c1495d8d1d930a046895c7 2024-11-10T09:26:21,649 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/C of 799270abaa9e80dc47bbc7cec6771c00 into 9c0539e123c1495d8d1d930a046895c7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:21,649 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:21,650 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/C, priority=11, startTime=1731230781563; duration=0sec 2024-11-10T09:26:21,650 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:21,650 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:C 2024-11-10T09:26:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:21,778 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:21,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:26:21,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c45d5003ac284894b7be62ca24c81a97 is 50, key is test_row_0/A:col10/1731230781777/Put/seqid=0 2024-11-10T09:26:21,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742096_1272 (size=14541) 2024-11-10T09:26:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230841793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230841793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230841794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230841794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T09:26:21,856 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:21,856 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T09:26:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:26:21,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:21,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:21,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:21,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230841906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230841906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230841906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230841906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:26:22,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T09:26:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:22,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:22,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230842116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230842116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230842116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230842116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T09:26:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:22,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:22,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:26:22,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c45d5003ac284894b7be62ca24c81a97 2024-11-10T09:26:22,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09 is 50, key is test_row_0/B:col10/1731230781777/Put/seqid=0 2024-11-10T09:26:22,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742097_1273 (size=12151) 2024-11-10T09:26:22,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09 2024-11-10T09:26:22,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/ef6fde3ad5ca4a4d8480c93aaecdbb92 is 50, key is test_row_0/C:col10/1731230781777/Put/seqid=0 2024-11-10T09:26:22,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742098_1274 (size=12151) 2024-11-10T09:26:22,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/ef6fde3ad5ca4a4d8480c93aaecdbb92 2024-11-10T09:26:22,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c45d5003ac284894b7be62ca24c81a97 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c45d5003ac284894b7be62ca24c81a97 2024-11-10T09:26:22,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c45d5003ac284894b7be62ca24c81a97, entries=200, sequenceid=205, filesize=14.2 K 2024-11-10T09:26:22,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09 2024-11-10T09:26:22,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09, entries=150, sequenceid=205, filesize=11.9 K 2024-11-10T09:26:22,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/ef6fde3ad5ca4a4d8480c93aaecdbb92 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ef6fde3ad5ca4a4d8480c93aaecdbb92 2024-11-10T09:26:22,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ef6fde3ad5ca4a4d8480c93aaecdbb92, entries=150, sequenceid=205, filesize=11.9 K 2024-11-10T09:26:22,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 799270abaa9e80dc47bbc7cec6771c00 in 456ms, sequenceid=205, compaction requested=false 2024-11-10T09:26:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:22,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T09:26:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:22,317 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:26:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/06d008506cb04269a4a042e8b04a09c5 is 50, key is test_row_0/A:col10/1731230781793/Put/seqid=0 2024-11-10T09:26:22,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742099_1275 (size=12151) 2024-11-10T09:26:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:22,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:22,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230842446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230842447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230842449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230842450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:26:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230842555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230842555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230842555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230842565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,725 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/06d008506cb04269a4a042e8b04a09c5 2024-11-10T09:26:22,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/f7713f8ff05a45879100416e974cc6bf is 50, key is test_row_0/B:col10/1731230781793/Put/seqid=0 2024-11-10T09:26:22,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742100_1276 (size=12151) 2024-11-10T09:26:22,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230842766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230842766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230842766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230842775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:26:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230843076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230843076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230843076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230843085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,137 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/f7713f8ff05a45879100416e974cc6bf 2024-11-10T09:26:23,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/262b9058ff044a78bfc360a7108f6367 is 50, key is test_row_0/C:col10/1731230781793/Put/seqid=0 2024-11-10T09:26:23,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742101_1277 (size=12151) 2024-11-10T09:26:23,147 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/262b9058ff044a78bfc360a7108f6367 2024-11-10T09:26:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/06d008506cb04269a4a042e8b04a09c5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/06d008506cb04269a4a042e8b04a09c5 2024-11-10T09:26:23,154 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/06d008506cb04269a4a042e8b04a09c5, entries=150, sequenceid=217, filesize=11.9 K 2024-11-10T09:26:23,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/f7713f8ff05a45879100416e974cc6bf as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f7713f8ff05a45879100416e974cc6bf 2024-11-10T09:26:23,158 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f7713f8ff05a45879100416e974cc6bf, entries=150, sequenceid=217, filesize=11.9 K 2024-11-10T09:26:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/262b9058ff044a78bfc360a7108f6367 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/262b9058ff044a78bfc360a7108f6367 2024-11-10T09:26:23,162 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/262b9058ff044a78bfc360a7108f6367, entries=150, sequenceid=217, filesize=11.9 K 2024-11-10T09:26:23,163 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 799270abaa9e80dc47bbc7cec6771c00 in 846ms, sequenceid=217, compaction requested=true 2024-11-10T09:26:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T09:26:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T09:26:23,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T09:26:23,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3060 sec 2024-11-10T09:26:23,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3100 sec 2024-11-10T09:26:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:23,588 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:23,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:26:23,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/75abcc69f1124b37b66df89cd3003242 is 50, key is test_row_0/A:col10/1731230783587/Put/seqid=0 2024-11-10T09:26:23,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742102_1278 (size=12151) 2024-11-10T09:26:23,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/75abcc69f1124b37b66df89cd3003242 2024-11-10T09:26:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230843597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/ed868a36b08a4196bea8fc65d6cce229 is 50, key is test_row_0/B:col10/1731230783587/Put/seqid=0 2024-11-10T09:26:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230843599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230843600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230843600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742103_1279 (size=12151) 2024-11-10T09:26:23,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/ed868a36b08a4196bea8fc65d6cce229 2024-11-10T09:26:23,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/1d3b567d7d4947a9972c6c3ab503918d is 50, key is test_row_0/C:col10/1731230783587/Put/seqid=0 2024-11-10T09:26:23,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742104_1280 (size=12151) 2024-11-10T09:26:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230843705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230843715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230843716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230843716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230843915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230843925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230843926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:23,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230843926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:23,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T09:26:24,006 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:24,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T09:26:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:26:24,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:24,008 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:24,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:24,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/1d3b567d7d4947a9972c6c3ab503918d 2024-11-10T09:26:24,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/75abcc69f1124b37b66df89cd3003242 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/75abcc69f1124b37b66df89cd3003242 2024-11-10T09:26:24,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/75abcc69f1124b37b66df89cd3003242, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T09:26:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/ed868a36b08a4196bea8fc65d6cce229 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ed868a36b08a4196bea8fc65d6cce229 2024-11-10T09:26:24,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ed868a36b08a4196bea8fc65d6cce229, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T09:26:24,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/1d3b567d7d4947a9972c6c3ab503918d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1d3b567d7d4947a9972c6c3ab503918d 2024-11-10T09:26:24,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1d3b567d7d4947a9972c6c3ab503918d, entries=150, sequenceid=242, filesize=11.9 K 2024-11-10T09:26:24,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 799270abaa9e80dc47bbc7cec6771c00 in 457ms, sequenceid=242, compaction requested=true 2024-11-10T09:26:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:24,045 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:26:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:24,045 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:26:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:24,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:24,046 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:26:24,046 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/A is initiating minor compaction (all files) 2024-11-10T09:26:24,046 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:26:24,046 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/B is initiating minor compaction (all files) 2024-11-10T09:26:24,046 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/A in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:24,046 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/B in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:24,046 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/fb5c1ec2813a47ea866e579497973564, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c45d5003ac284894b7be62ca24c81a97, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/06d008506cb04269a4a042e8b04a09c5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/75abcc69f1124b37b66df89cd3003242] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=50.2 K 2024-11-10T09:26:24,046 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/480ce0ad444d416886328b87f22f2ea4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f7713f8ff05a45879100416e974cc6bf, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ed868a36b08a4196bea8fc65d6cce229] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=47.9 K 2024-11-10T09:26:24,047 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb5c1ec2813a47ea866e579497973564, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731230779968 2024-11-10T09:26:24,047 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 480ce0ad444d416886328b87f22f2ea4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731230779968 2024-11-10T09:26:24,047 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45d5003ac284894b7be62ca24c81a97, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731230781142 2024-11-10T09:26:24,047 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a3fb1fa0c1a44ae580c2b6c7ae8a9f09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731230781142 2024-11-10T09:26:24,047 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d008506cb04269a4a042e8b04a09c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230781781 2024-11-10T09:26:24,047 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f7713f8ff05a45879100416e974cc6bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230781781 2024-11-10T09:26:24,047 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75abcc69f1124b37b66df89cd3003242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731230782444 2024-11-10T09:26:24,048 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ed868a36b08a4196bea8fc65d6cce229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731230782444 2024-11-10T09:26:24,055 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:24,056 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/e0940e82acbc45c58e4efb18f71cfabf is 50, key is test_row_0/B:col10/1731230783587/Put/seqid=0 2024-11-10T09:26:24,063 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:24,063 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/47b07082b36947da94b3ba9cd37a34d0 is 50, key is test_row_0/A:col10/1731230783587/Put/seqid=0 2024-11-10T09:26:24,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742105_1281 (size=12697) 2024-11-10T09:26:24,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742106_1282 (size=12697) 2024-11-10T09:26:24,079 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/47b07082b36947da94b3ba9cd37a34d0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/47b07082b36947da94b3ba9cd37a34d0 2024-11-10T09:26:24,084 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/A of 799270abaa9e80dc47bbc7cec6771c00 into 47b07082b36947da94b3ba9cd37a34d0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:24,084 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:24,085 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/A, priority=12, startTime=1731230784045; duration=0sec 2024-11-10T09:26:24,085 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:24,085 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:A 2024-11-10T09:26:24,085 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:26:24,086 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:26:24,086 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/C is initiating minor compaction (all files) 2024-11-10T09:26:24,086 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/C in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:24,086 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9c0539e123c1495d8d1d930a046895c7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ef6fde3ad5ca4a4d8480c93aaecdbb92, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/262b9058ff044a78bfc360a7108f6367, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1d3b567d7d4947a9972c6c3ab503918d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=47.9 K 2024-11-10T09:26:24,087 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c0539e123c1495d8d1d930a046895c7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731230779968 2024-11-10T09:26:24,087 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6fde3ad5ca4a4d8480c93aaecdbb92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731230781142 2024-11-10T09:26:24,087 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262b9058ff044a78bfc360a7108f6367, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731230781781 2024-11-10T09:26:24,087 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3b567d7d4947a9972c6c3ab503918d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731230782444 2024-11-10T09:26:24,096 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#C#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:24,100 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/e9c43ce142f24882a5f98e0eab420396 is 50, key is test_row_0/C:col10/1731230783587/Put/seqid=0 2024-11-10T09:26:24,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742107_1283 (size=12697) 2024-11-10T09:26:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:26:24,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T09:26:24,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:24,160 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:26:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c303415db548407e9ade0be9783321af is 50, key is test_row_0/A:col10/1731230783593/Put/seqid=0 2024-11-10T09:26:24,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742108_1284 (size=12151) 2024-11-10T09:26:24,169 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c303415db548407e9ade0be9783321af 2024-11-10T09:26:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/fd6a2e06f40a4e27b887cf53d6e1c461 is 50, key is test_row_0/B:col10/1731230783593/Put/seqid=0 2024-11-10T09:26:24,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742109_1285 (size=12151) 2024-11-10T09:26:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:24,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:24,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230844259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230844260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230844261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230844263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:26:24,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230844366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230844375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230844375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230844375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,474 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/e0940e82acbc45c58e4efb18f71cfabf as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e0940e82acbc45c58e4efb18f71cfabf 2024-11-10T09:26:24,478 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/B of 799270abaa9e80dc47bbc7cec6771c00 into e0940e82acbc45c58e4efb18f71cfabf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:24,478 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:24,478 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/B, priority=12, startTime=1731230784045; duration=0sec 2024-11-10T09:26:24,478 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:24,478 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:B 2024-11-10T09:26:24,508 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/e9c43ce142f24882a5f98e0eab420396 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e9c43ce142f24882a5f98e0eab420396 2024-11-10T09:26:24,515 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/C of 799270abaa9e80dc47bbc7cec6771c00 into e9c43ce142f24882a5f98e0eab420396(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:24,515 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:24,515 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/C, priority=12, startTime=1731230784045; duration=0sec 2024-11-10T09:26:24,515 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:24,515 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:C 2024-11-10T09:26:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230844576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,581 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/fd6a2e06f40a4e27b887cf53d6e1c461 2024-11-10T09:26:24,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/f357182d114f4294a31dfa162be3e184 is 50, key is test_row_0/C:col10/1731230783593/Put/seqid=0 2024-11-10T09:26:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230844586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230844586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230844586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742110_1286 (size=12151) 2024-11-10T09:26:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:26:24,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230844885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230844896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230844896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230844896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:24,992 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/f357182d114f4294a31dfa162be3e184 2024-11-10T09:26:24,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c303415db548407e9ade0be9783321af as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c303415db548407e9ade0be9783321af 2024-11-10T09:26:25,001 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c303415db548407e9ade0be9783321af, entries=150, sequenceid=255, filesize=11.9 K 2024-11-10T09:26:25,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/fd6a2e06f40a4e27b887cf53d6e1c461 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd6a2e06f40a4e27b887cf53d6e1c461 2024-11-10T09:26:25,006 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd6a2e06f40a4e27b887cf53d6e1c461, entries=150, sequenceid=255, filesize=11.9 K 2024-11-10T09:26:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/f357182d114f4294a31dfa162be3e184 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f357182d114f4294a31dfa162be3e184 2024-11-10T09:26:25,011 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f357182d114f4294a31dfa162be3e184, entries=150, sequenceid=255, filesize=11.9 K 2024-11-10T09:26:25,012 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 799270abaa9e80dc47bbc7cec6771c00 in 852ms, sequenceid=255, compaction requested=false 2024-11-10T09:26:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:25,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T09:26:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T09:26:25,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T09:26:25,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-10T09:26:25,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0100 sec 2024-11-10T09:26:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T09:26:25,146 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:25,146 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T09:26:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:26:25,148 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:25,148 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:25,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:26:25,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T09:26:25,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:25,300 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:26:25,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/9e8a64bd2e1e4999b1cbbc2acc74a028 is 50, key is test_row_0/A:col10/1731230784261/Put/seqid=0 2024-11-10T09:26:25,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742111_1287 (size=12301) 2024-11-10T09:26:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:25,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:25,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230845411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230845442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230845442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230845442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:26:25,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230845547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230845556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230845556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230845556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,711 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/9e8a64bd2e1e4999b1cbbc2acc74a028 2024-11-10T09:26:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/46cd739687e2401299e3fd7b6f26bd4e is 50, key is test_row_0/B:col10/1731230784261/Put/seqid=0 2024-11-10T09:26:25,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742112_1288 (size=12301) 2024-11-10T09:26:25,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230845756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230845766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230845766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230845766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:26:26,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230846065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230846076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230846076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230846076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,121 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/46cd739687e2401299e3fd7b6f26bd4e 2024-11-10T09:26:26,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/a95ceeb5938e499caf2b252a012f2a00 is 50, key is test_row_0/C:col10/1731230784261/Put/seqid=0 2024-11-10T09:26:26,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742113_1289 (size=12301) 2024-11-10T09:26:26,132 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/a95ceeb5938e499caf2b252a012f2a00 2024-11-10T09:26:26,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/9e8a64bd2e1e4999b1cbbc2acc74a028 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9e8a64bd2e1e4999b1cbbc2acc74a028 2024-11-10T09:26:26,147 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9e8a64bd2e1e4999b1cbbc2acc74a028, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T09:26:26,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/46cd739687e2401299e3fd7b6f26bd4e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/46cd739687e2401299e3fd7b6f26bd4e 2024-11-10T09:26:26,152 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/46cd739687e2401299e3fd7b6f26bd4e, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T09:26:26,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/a95ceeb5938e499caf2b252a012f2a00 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a95ceeb5938e499caf2b252a012f2a00 2024-11-10T09:26:26,156 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a95ceeb5938e499caf2b252a012f2a00, entries=150, sequenceid=281, filesize=12.0 K 2024-11-10T09:26:26,157 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 799270abaa9e80dc47bbc7cec6771c00 in 857ms, sequenceid=281, compaction requested=true 2024-11-10T09:26:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T09:26:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T09:26:26,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T09:26:26,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-10T09:26:26,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0150 sec 2024-11-10T09:26:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T09:26:26,286 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:26,286 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-10T09:26:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:26:26,288 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:26,289 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:26,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:26:26,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T09:26:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:26,441 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:26:26,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/ef34f42a38914023afaa4c1bf276d78c is 50, key is test_row_0/A:col10/1731230785413/Put/seqid=0 2024-11-10T09:26:26,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742114_1290 (size=12301) 2024-11-10T09:26:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:26,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:26:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230846607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230846608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230846611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230846612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230846715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230846715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230846726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230846726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,848 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/ef34f42a38914023afaa4c1bf276d78c 2024-11-10T09:26:26,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3d1d29bf7c0a4ce69a037ffb22844041 is 50, key is test_row_0/B:col10/1731230785413/Put/seqid=0 2024-11-10T09:26:26,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742115_1291 (size=12301) 2024-11-10T09:26:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:26:26,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230846925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230846925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230846936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230846936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230847235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230847235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230847245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230847245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,259 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3d1d29bf7c0a4ce69a037ffb22844041 2024-11-10T09:26:27,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/cd34803864854069b22b2180d63215b1 is 50, key is test_row_0/C:col10/1731230785413/Put/seqid=0 2024-11-10T09:26:27,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742116_1292 (size=12301) 2024-11-10T09:26:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:26:27,668 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/cd34803864854069b22b2180d63215b1 2024-11-10T09:26:27,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/ef34f42a38914023afaa4c1bf276d78c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ef34f42a38914023afaa4c1bf276d78c 2024-11-10T09:26:27,675 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ef34f42a38914023afaa4c1bf276d78c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T09:26:27,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3d1d29bf7c0a4ce69a037ffb22844041 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3d1d29bf7c0a4ce69a037ffb22844041 2024-11-10T09:26:27,679 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3d1d29bf7c0a4ce69a037ffb22844041, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T09:26:27,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/cd34803864854069b22b2180d63215b1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/cd34803864854069b22b2180d63215b1 2024-11-10T09:26:27,683 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/cd34803864854069b22b2180d63215b1, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T09:26:27,684 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 799270abaa9e80dc47bbc7cec6771c00 in 1243ms, sequenceid=294, compaction requested=true 2024-11-10T09:26:27,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:27,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:27,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-10T09:26:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-10T09:26:27,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T09:26:27,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-10T09:26:27,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.4010 sec 2024-11-10T09:26:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:27,749 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:27,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:26:27,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c7b333b615824ce08d6be453d1a979d3 is 50, key is test_row_0/A:col10/1731230787748/Put/seqid=0 2024-11-10T09:26:27,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742117_1293 (size=14741) 2024-11-10T09:26:27,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230847761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230847761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230847762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230847764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230847875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230847876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230847876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230847876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:27,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230848085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230848086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230848086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230848086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c7b333b615824ce08d6be453d1a979d3 2024-11-10T09:26:28,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/510417a7f6524e35aef218603e2a43e6 is 50, key is test_row_0/B:col10/1731230787748/Put/seqid=0 2024-11-10T09:26:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742118_1294 (size=12301) 2024-11-10T09:26:28,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230848395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230848395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230848395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230848395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T09:26:28,436 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:28,437 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-10T09:26:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T09:26:28,438 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:28,439 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:28,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T09:26:28,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/510417a7f6524e35aef218603e2a43e6 2024-11-10T09:26:28,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/4d128029a1d244ae8fbe2a0355f59f06 is 50, key is test_row_0/C:col10/1731230787748/Put/seqid=0 2024-11-10T09:26:28,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T09:26:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:28,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742119_1295 (size=12301) 2024-11-10T09:26:28,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T09:26:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:28,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T09:26:28,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T09:26:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:28,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230848905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230848905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230848906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230848906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:28,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:28,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/4d128029a1d244ae8fbe2a0355f59f06 2024-11-10T09:26:28,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c7b333b615824ce08d6be453d1a979d3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c7b333b615824ce08d6be453d1a979d3 2024-11-10T09:26:29,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c7b333b615824ce08d6be453d1a979d3, entries=200, sequenceid=319, filesize=14.4 K 2024-11-10T09:26:29,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/510417a7f6524e35aef218603e2a43e6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/510417a7f6524e35aef218603e2a43e6 2024-11-10T09:26:29,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/510417a7f6524e35aef218603e2a43e6, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T09:26:29,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/4d128029a1d244ae8fbe2a0355f59f06 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/4d128029a1d244ae8fbe2a0355f59f06 2024-11-10T09:26:29,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/4d128029a1d244ae8fbe2a0355f59f06, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T09:26:29,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 799270abaa9e80dc47bbc7cec6771c00 in 1263ms, sequenceid=319, compaction requested=true 2024-11-10T09:26:29,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:29,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:29,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:29,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:29,013 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:26:29,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:29,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:29,013 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:26:29,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:29,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:26:29,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/B is initiating minor compaction (all files) 2024-11-10T09:26:29,014 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/B in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:29,014 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e0940e82acbc45c58e4efb18f71cfabf, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd6a2e06f40a4e27b887cf53d6e1c461, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/46cd739687e2401299e3fd7b6f26bd4e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3d1d29bf7c0a4ce69a037ffb22844041, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/510417a7f6524e35aef218603e2a43e6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=60.3 K 2024-11-10T09:26:29,014 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64191 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:26:29,014 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/A is initiating minor compaction (all files) 2024-11-10T09:26:29,014 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/A in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:29,014 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/47b07082b36947da94b3ba9cd37a34d0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c303415db548407e9ade0be9783321af, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9e8a64bd2e1e4999b1cbbc2acc74a028, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ef34f42a38914023afaa4c1bf276d78c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c7b333b615824ce08d6be453d1a979d3] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=62.7 K 2024-11-10T09:26:29,015 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47b07082b36947da94b3ba9cd37a34d0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731230782444 2024-11-10T09:26:29,015 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e0940e82acbc45c58e4efb18f71cfabf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731230782444 2024-11-10T09:26:29,015 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c303415db548407e9ade0be9783321af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230783593 2024-11-10T09:26:29,015 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting fd6a2e06f40a4e27b887cf53d6e1c461, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230783593 2024-11-10T09:26:29,015 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e8a64bd2e1e4999b1cbbc2acc74a028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731230784259 2024-11-10T09:26:29,015 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 46cd739687e2401299e3fd7b6f26bd4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731230784259 2024-11-10T09:26:29,016 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef34f42a38914023afaa4c1bf276d78c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731230785409 2024-11-10T09:26:29,016 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d1d29bf7c0a4ce69a037ffb22844041, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731230785409 2024-11-10T09:26:29,016 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b333b615824ce08d6be453d1a979d3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230786604 2024-11-10T09:26:29,016 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 510417a7f6524e35aef218603e2a43e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230786604 2024-11-10T09:26:29,025 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#A#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:29,026 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/3019aad774e74639824a4d7393e501fb is 50, key is test_row_0/A:col10/1731230787748/Put/seqid=0 2024-11-10T09:26:29,028 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:29,029 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3b76a2a6fa6b4bd9b978d15d1935603d is 50, key is test_row_0/B:col10/1731230787748/Put/seqid=0 2024-11-10T09:26:29,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742120_1296 (size=13017) 2024-11-10T09:26:29,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742121_1297 (size=13017) 2024-11-10T09:26:29,037 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/3b76a2a6fa6b4bd9b978d15d1935603d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3b76a2a6fa6b4bd9b978d15d1935603d 2024-11-10T09:26:29,041 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/B of 799270abaa9e80dc47bbc7cec6771c00 into 3b76a2a6fa6b4bd9b978d15d1935603d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:29,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:29,041 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/B, priority=11, startTime=1731230789012; duration=0sec 2024-11-10T09:26:29,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:29,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:B 2024-11-10T09:26:29,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T09:26:29,042 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T09:26:29,043 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/C is initiating minor compaction (all files) 2024-11-10T09:26:29,043 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/C in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:29,043 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e9c43ce142f24882a5f98e0eab420396, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f357182d114f4294a31dfa162be3e184, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a95ceeb5938e499caf2b252a012f2a00, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/cd34803864854069b22b2180d63215b1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/4d128029a1d244ae8fbe2a0355f59f06] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=60.3 K 2024-11-10T09:26:29,043 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c43ce142f24882a5f98e0eab420396, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731230782444 2024-11-10T09:26:29,043 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f357182d114f4294a31dfa162be3e184, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230783593 2024-11-10T09:26:29,044 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a95ceeb5938e499caf2b252a012f2a00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731230784259 2024-11-10T09:26:29,044 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting cd34803864854069b22b2180d63215b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731230785409 2024-11-10T09:26:29,044 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d128029a1d244ae8fbe2a0355f59f06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230786604 2024-11-10T09:26:29,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-10T09:26:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:29,050 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:26:29,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/f6379fb14d874e1c995ce3648a581c27 is 50, key is test_row_0/A:col10/1731230787759/Put/seqid=0 2024-11-10T09:26:29,057 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#C#compaction#257 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:29,057 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/1eb6112b4a194b71a01b847f0a2a3c60 is 50, key is test_row_0/C:col10/1731230787748/Put/seqid=0 2024-11-10T09:26:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T09:26:29,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742122_1298 (size=12301) 2024-11-10T09:26:29,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742123_1299 (size=13017) 2024-11-10T09:26:29,074 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/f6379fb14d874e1c995ce3648a581c27 2024-11-10T09:26:29,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/e2e20b4649ab4392a86a5b02f0d76b14 is 50, key is test_row_0/B:col10/1731230787759/Put/seqid=0 2024-11-10T09:26:29,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742124_1300 (size=12301) 2024-11-10T09:26:29,436 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/3019aad774e74639824a4d7393e501fb as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3019aad774e74639824a4d7393e501fb 2024-11-10T09:26:29,441 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/A of 799270abaa9e80dc47bbc7cec6771c00 into 3019aad774e74639824a4d7393e501fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:29,441 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:29,441 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/A, priority=11, startTime=1731230789012; duration=0sec 2024-11-10T09:26:29,441 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:29,441 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:A 2024-11-10T09:26:29,477 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/1eb6112b4a194b71a01b847f0a2a3c60 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1eb6112b4a194b71a01b847f0a2a3c60 2024-11-10T09:26:29,482 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/C of 799270abaa9e80dc47bbc7cec6771c00 into 1eb6112b4a194b71a01b847f0a2a3c60(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:29,482 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:29,482 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/C, priority=11, startTime=1731230789013; duration=0sec 2024-11-10T09:26:29,482 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:29,482 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:C 2024-11-10T09:26:29,494 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/e2e20b4649ab4392a86a5b02f0d76b14 2024-11-10T09:26:29,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/61939711084b41ea87a219fecf89f2cb is 50, key is test_row_0/C:col10/1731230787759/Put/seqid=0 2024-11-10T09:26:29,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742125_1301 (size=12301) 2024-11-10T09:26:29,505 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/61939711084b41ea87a219fecf89f2cb 2024-11-10T09:26:29,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/f6379fb14d874e1c995ce3648a581c27 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/f6379fb14d874e1c995ce3648a581c27 2024-11-10T09:26:29,513 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/f6379fb14d874e1c995ce3648a581c27, entries=150, sequenceid=331, filesize=12.0 K 2024-11-10T09:26:29,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/e2e20b4649ab4392a86a5b02f0d76b14 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e2e20b4649ab4392a86a5b02f0d76b14 2024-11-10T09:26:29,517 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e2e20b4649ab4392a86a5b02f0d76b14, entries=150, sequenceid=331, filesize=12.0 K 2024-11-10T09:26:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/61939711084b41ea87a219fecf89f2cb as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/61939711084b41ea87a219fecf89f2cb 2024-11-10T09:26:29,522 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/61939711084b41ea87a219fecf89f2cb, entries=150, sequenceid=331, filesize=12.0 K 2024-11-10T09:26:29,523 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 799270abaa9e80dc47bbc7cec6771c00 in 474ms, sequenceid=331, compaction requested=false 2024-11-10T09:26:29,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:29,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:29,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-10T09:26:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-10T09:26:29,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T09:26:29,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-10T09:26:29,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.0900 sec 2024-11-10T09:26:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T09:26:29,576 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:29,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-10T09:26:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T09:26:29,578 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:29,578 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:29,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T09:26:29,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-10T09:26:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:29,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-10T09:26:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-10T09:26:29,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-10T09:26:29,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-10T09:26:29,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 158 msec 2024-11-10T09:26:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T09:26:29,895 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:29,895 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-10T09:26:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T09:26:29,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:29,898 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:29,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:29,921 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:29,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:26:29,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/3833f506d84649b4a1e8fb253f6d419f is 50, key is test_row_0/A:col10/1731230789920/Put/seqid=0 2024-11-10T09:26:29,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742126_1302 (size=14741) 2024-11-10T09:26:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230849945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230849945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:29,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230849947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:29,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230849948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T09:26:30,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T09:26:30,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:30,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230850056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230850056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230850056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230850056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T09:26:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T09:26:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230850266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230850266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230850266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230850266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/3833f506d84649b4a1e8fb253f6d419f 2024-11-10T09:26:30,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/612a3808e2264a1bbe361bcb25293c1f is 50, key is test_row_0/B:col10/1731230789920/Put/seqid=0 2024-11-10T09:26:30,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742127_1303 (size=12301) 2024-11-10T09:26:30,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/612a3808e2264a1bbe361bcb25293c1f 2024-11-10T09:26:30,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/771df78f22c8447baa1d7f54b08d71f7 is 50, key is test_row_0/C:col10/1731230789920/Put/seqid=0 2024-11-10T09:26:30,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T09:26:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742128_1304 (size=12301) 2024-11-10T09:26:30,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40984 deadline: 1731230850405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,408 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 799270abaa9e80dc47bbc7cec6771c00 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T09:26:30,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T09:26:30,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230850576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230850576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230850576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230850576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:30,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:30,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T09:26:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:30,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/771df78f22c8447baa1d7f54b08d71f7 2024-11-10T09:26:30,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/3833f506d84649b4a1e8fb253f6d419f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3833f506d84649b4a1e8fb253f6d419f 2024-11-10T09:26:30,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3833f506d84649b4a1e8fb253f6d419f, entries=200, sequenceid=346, filesize=14.4 K 2024-11-10T09:26:30,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/612a3808e2264a1bbe361bcb25293c1f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/612a3808e2264a1bbe361bcb25293c1f 2024-11-10T09:26:30,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/612a3808e2264a1bbe361bcb25293c1f, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T09:26:30,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/771df78f22c8447baa1d7f54b08d71f7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/771df78f22c8447baa1d7f54b08d71f7 2024-11-10T09:26:30,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/771df78f22c8447baa1d7f54b08d71f7, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T09:26:30,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 799270abaa9e80dc47bbc7cec6771c00 in 853ms, sequenceid=346, compaction requested=true 2024-11-10T09:26:30,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:30,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:30,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:30,774 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:30,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:30,775 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:30,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:30,775 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:30,775 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:30,775 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/B is initiating minor compaction (all files) 2024-11-10T09:26:30,775 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/A is initiating minor compaction (all files) 2024-11-10T09:26:30,776 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/A in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,776 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/B in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,776 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3019aad774e74639824a4d7393e501fb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/f6379fb14d874e1c995ce3648a581c27, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3833f506d84649b4a1e8fb253f6d419f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=39.1 K 2024-11-10T09:26:30,776 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3b76a2a6fa6b4bd9b978d15d1935603d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e2e20b4649ab4392a86a5b02f0d76b14, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/612a3808e2264a1bbe361bcb25293c1f] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=36.7 K 2024-11-10T09:26:30,776 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3019aad774e74639824a4d7393e501fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230786604 2024-11-10T09:26:30,776 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b76a2a6fa6b4bd9b978d15d1935603d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230786604 2024-11-10T09:26:30,776 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e20b4649ab4392a86a5b02f0d76b14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731230787759 2024-11-10T09:26:30,776 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6379fb14d874e1c995ce3648a581c27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731230787759 2024-11-10T09:26:30,776 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3833f506d84649b4a1e8fb253f6d419f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731230789917 2024-11-10T09:26:30,776 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 612a3808e2264a1bbe361bcb25293c1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731230789920 2024-11-10T09:26:30,783 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#B#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:30,784 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/483e00d03a7a49bd8c3935802d180de1 is 50, key is test_row_0/B:col10/1731230789920/Put/seqid=0 2024-11-10T09:26:30,790 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:30,791 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/8a237ab1e2784e73aed08c39e50a03c7 is 50, key is test_row_0/A:col10/1731230789920/Put/seqid=0 2024-11-10T09:26:30,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742130_1306 (size=13119) 2024-11-10T09:26:30,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742129_1305 (size=13119) 2024-11-10T09:26:30,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-10T09:26:30,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:30,814 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:26:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/6ec4a19ba26f4ce087dde9dcfad4d210 is 50, key is test_row_0/A:col10/1731230789944/Put/seqid=0 2024-11-10T09:26:30,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742131_1307 (size=12301) 2024-11-10T09:26:30,822 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/6ec4a19ba26f4ce087dde9dcfad4d210 2024-11-10T09:26:30,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/d36feed65b6f4221835055b01fab057a is 50, key is test_row_0/B:col10/1731230789944/Put/seqid=0 2024-11-10T09:26:30,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742132_1308 (size=12301) 2024-11-10T09:26:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T09:26:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:31,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:31,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230851093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230851093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230851094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230851094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,200 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/8a237ab1e2784e73aed08c39e50a03c7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/8a237ab1e2784e73aed08c39e50a03c7 2024-11-10T09:26:31,200 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/483e00d03a7a49bd8c3935802d180de1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/483e00d03a7a49bd8c3935802d180de1 2024-11-10T09:26:31,204 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/A of 799270abaa9e80dc47bbc7cec6771c00 into 8a237ab1e2784e73aed08c39e50a03c7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:31,204 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/B of 799270abaa9e80dc47bbc7cec6771c00 into 483e00d03a7a49bd8c3935802d180de1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:31,204 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:31,204 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:31,204 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/B, priority=13, startTime=1731230790774; duration=0sec 2024-11-10T09:26:31,204 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/A, priority=13, startTime=1731230790774; duration=0sec 2024-11-10T09:26:31,204 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:31,204 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:A 2024-11-10T09:26:31,204 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:31,204 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:B 2024-11-10T09:26:31,204 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:31,205 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:31,205 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/C is initiating minor compaction (all files) 2024-11-10T09:26:31,206 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/C in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:31,206 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1eb6112b4a194b71a01b847f0a2a3c60, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/61939711084b41ea87a219fecf89f2cb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/771df78f22c8447baa1d7f54b08d71f7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=36.7 K 2024-11-10T09:26:31,206 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eb6112b4a194b71a01b847f0a2a3c60, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731230786604 2024-11-10T09:26:31,206 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61939711084b41ea87a219fecf89f2cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731230787759 2024-11-10T09:26:31,206 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 771df78f22c8447baa1d7f54b08d71f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731230789920 2024-11-10T09:26:31,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230851205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230851205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230851205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230851205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,213 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#C#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:31,214 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/58f838de55aa462eb2790b93de646235 is 50, key is test_row_0/C:col10/1731230789920/Put/seqid=0 2024-11-10T09:26:31,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742133_1309 (size=13119) 2024-11-10T09:26:31,233 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/d36feed65b6f4221835055b01fab057a 2024-11-10T09:26:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/5bd255903f7d4ed8b176aa54667f9a37 is 50, key is test_row_0/C:col10/1731230789944/Put/seqid=0 2024-11-10T09:26:31,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742134_1310 (size=12301) 2024-11-10T09:26:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230851415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230851415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230851415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230851415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,622 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/58f838de55aa462eb2790b93de646235 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58f838de55aa462eb2790b93de646235 2024-11-10T09:26:31,626 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/C of 799270abaa9e80dc47bbc7cec6771c00 into 58f838de55aa462eb2790b93de646235(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:31,626 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:31,626 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/C, priority=13, startTime=1731230790775; duration=0sec 2024-11-10T09:26:31,626 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:31,626 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:C 2024-11-10T09:26:31,647 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/5bd255903f7d4ed8b176aa54667f9a37 2024-11-10T09:26:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/6ec4a19ba26f4ce087dde9dcfad4d210 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6ec4a19ba26f4ce087dde9dcfad4d210 2024-11-10T09:26:31,654 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6ec4a19ba26f4ce087dde9dcfad4d210, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:26:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/d36feed65b6f4221835055b01fab057a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d36feed65b6f4221835055b01fab057a 2024-11-10T09:26:31,659 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d36feed65b6f4221835055b01fab057a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:26:31,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/5bd255903f7d4ed8b176aa54667f9a37 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/5bd255903f7d4ed8b176aa54667f9a37 2024-11-10T09:26:31,663 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/5bd255903f7d4ed8b176aa54667f9a37, entries=150, sequenceid=371, filesize=12.0 K 2024-11-10T09:26:31,663 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 799270abaa9e80dc47bbc7cec6771c00 in 849ms, sequenceid=371, compaction requested=false 2024-11-10T09:26:31,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:31,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:31,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-10T09:26:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-10T09:26:31,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-10T09:26:31,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7670 sec 2024-11-10T09:26:31,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.7710 sec 2024-11-10T09:26:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:31,728 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:31,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:26:31,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/7c4fbdc493db4ce4b9f43d5ce5e55125 is 50, key is test_row_0/A:col10/1731230791093/Put/seqid=0 2024-11-10T09:26:31,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742135_1311 (size=14741) 2024-11-10T09:26:31,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230851752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230851752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230851756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230851757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230851865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230851865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230851865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230851865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T09:26:32,046 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:32,047 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-10T09:26:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T09:26:32,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:32,049 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:32,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230852075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230852075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230852075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230852076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,123 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-10T09:26:32,123 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:32,123 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,123 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:32,124 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-10T09:26:32,124 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:32,124 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,124 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:32,124 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-10T09:26:32,124 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:32,124 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,125 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:32,126 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-10T09:26:32,126 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:32,126 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,126 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,126 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:32,127 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-10T09:26:32,127 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:32,127 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,127 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:32,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:32,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/7c4fbdc493db4ce4b9f43d5ce5e55125 2024-11-10T09:26:32,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/13cf2f1f6c564916942858982708fd75 is 50, key is test_row_0/B:col10/1731230791093/Put/seqid=0 2024-11-10T09:26:32,146 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:26:32,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742136_1312 (size=12301) 2024-11-10T09:26:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T09:26:32,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T09:26:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T09:26:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:32,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T09:26:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230852386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230852386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230852386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230852386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T09:26:32,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:32,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/13cf2f1f6c564916942858982708fd75 2024-11-10T09:26:32,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/d9db1ce35a614a52ac24d89e352b4a1b is 50, key is test_row_0/C:col10/1731230791093/Put/seqid=0 2024-11-10T09:26:32,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742137_1313 (size=12301) 2024-11-10T09:26:32,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T09:26:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T09:26:32,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T09:26:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41000 deadline: 1731230852896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40958 deadline: 1731230852896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40974 deadline: 1731230852896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41002 deadline: 1731230852896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:32,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:26:32,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/d9db1ce35a614a52ac24d89e352b4a1b 2024-11-10T09:26:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/7c4fbdc493db4ce4b9f43d5ce5e55125 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/7c4fbdc493db4ce4b9f43d5ce5e55125 2024-11-10T09:26:32,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T09:26:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/7c4fbdc493db4ce4b9f43d5ce5e55125, entries=200, sequenceid=388, filesize=14.4 K 2024-11-10T09:26:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:32,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/13cf2f1f6c564916942858982708fd75 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/13cf2f1f6c564916942858982708fd75 2024-11-10T09:26:32,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/13cf2f1f6c564916942858982708fd75, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T09:26:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/d9db1ce35a614a52ac24d89e352b4a1b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/d9db1ce35a614a52ac24d89e352b4a1b 2024-11-10T09:26:32,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/d9db1ce35a614a52ac24d89e352b4a1b, entries=150, sequenceid=388, filesize=12.0 K 2024-11-10T09:26:32,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 799270abaa9e80dc47bbc7cec6771c00 in 1261ms, sequenceid=388, compaction requested=true 2024-11-10T09:26:32,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:32,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:32,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:32,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:32,989 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:32,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:32,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 799270abaa9e80dc47bbc7cec6771c00:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:32,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:32,989 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:32,990 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:32,990 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:32,990 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/B is initiating minor compaction (all files) 2024-11-10T09:26:32,990 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/A is initiating minor compaction (all files) 2024-11-10T09:26:32,990 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/B in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,990 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/A in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:32,990 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/483e00d03a7a49bd8c3935802d180de1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d36feed65b6f4221835055b01fab057a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/13cf2f1f6c564916942858982708fd75] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=36.8 K 2024-11-10T09:26:32,990 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/8a237ab1e2784e73aed08c39e50a03c7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6ec4a19ba26f4ce087dde9dcfad4d210, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/7c4fbdc493db4ce4b9f43d5ce5e55125] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=39.2 K 2024-11-10T09:26:32,990 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a237ab1e2784e73aed08c39e50a03c7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731230789920 2024-11-10T09:26:32,990 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 483e00d03a7a49bd8c3935802d180de1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731230789920 2024-11-10T09:26:32,991 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ec4a19ba26f4ce087dde9dcfad4d210, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230789944 2024-11-10T09:26:32,991 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d36feed65b6f4221835055b01fab057a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230789944 2024-11-10T09:26:32,991 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c4fbdc493db4ce4b9f43d5ce5e55125, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731230791089 2024-11-10T09:26:32,991 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 13cf2f1f6c564916942858982708fd75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731230791093 2024-11-10T09:26:32,997 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#B#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:32,998 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/85e257b48d9d4e8ebe510457b4253b2a is 50, key is test_row_0/B:col10/1731230791093/Put/seqid=0 2024-11-10T09:26:32,998 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:32,999 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/12a15ea44e3e42ba99d44baee80b2a19 is 50, key is test_row_0/A:col10/1731230791093/Put/seqid=0 2024-11-10T09:26:33,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742138_1314 (size=13221) 2024-11-10T09:26:33,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742139_1315 (size=13221) 2024-11-10T09:26:33,007 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/85e257b48d9d4e8ebe510457b4253b2a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/85e257b48d9d4e8ebe510457b4253b2a 2024-11-10T09:26:33,011 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/B of 799270abaa9e80dc47bbc7cec6771c00 into 85e257b48d9d4e8ebe510457b4253b2a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:33,011 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:33,011 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/B, priority=13, startTime=1731230792989; duration=0sec 2024-11-10T09:26:33,011 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:33,011 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:B 2024-11-10T09:26:33,011 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:33,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:33,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 799270abaa9e80dc47bbc7cec6771c00/C is initiating minor compaction (all files) 2024-11-10T09:26:33,012 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 799270abaa9e80dc47bbc7cec6771c00/C in TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:33,012 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58f838de55aa462eb2790b93de646235, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/5bd255903f7d4ed8b176aa54667f9a37, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/d9db1ce35a614a52ac24d89e352b4a1b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp, totalSize=36.8 K 2024-11-10T09:26:33,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f838de55aa462eb2790b93de646235, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731230789920 2024-11-10T09:26:33,013 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd255903f7d4ed8b176aa54667f9a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731230789944 2024-11-10T09:26:33,013 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d9db1ce35a614a52ac24d89e352b4a1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731230791093 2024-11-10T09:26:33,019 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 799270abaa9e80dc47bbc7cec6771c00#C#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:33,020 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/e0b16bb4d46a4d4b8f9812388fff7b6d is 50, key is test_row_0/C:col10/1731230791093/Put/seqid=0 2024-11-10T09:26:33,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742140_1316 (size=13221) 2024-11-10T09:26:33,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-10T09:26:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:33,131 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:26:33,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c837c3ac6e4d47bab78b5e6dfea348b0 is 50, key is test_row_0/A:col10/1731230791756/Put/seqid=0 2024-11-10T09:26:33,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742141_1317 (size=12301) 2024-11-10T09:26:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T09:26:33,408 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/12a15ea44e3e42ba99d44baee80b2a19 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/12a15ea44e3e42ba99d44baee80b2a19 2024-11-10T09:26:33,413 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/A of 799270abaa9e80dc47bbc7cec6771c00 into 12a15ea44e3e42ba99d44baee80b2a19(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:33,413 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:33,413 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/A, priority=13, startTime=1731230792989; duration=0sec 2024-11-10T09:26:33,413 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:33,413 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:A 2024-11-10T09:26:33,429 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/e0b16bb4d46a4d4b8f9812388fff7b6d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e0b16bb4d46a4d4b8f9812388fff7b6d 2024-11-10T09:26:33,435 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 799270abaa9e80dc47bbc7cec6771c00/C of 799270abaa9e80dc47bbc7cec6771c00 into e0b16bb4d46a4d4b8f9812388fff7b6d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:33,435 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:33,435 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., storeName=799270abaa9e80dc47bbc7cec6771c00/C, priority=13, startTime=1731230792989; duration=0sec 2024-11-10T09:26:33,435 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:33,435 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 799270abaa9e80dc47bbc7cec6771c00:C 2024-11-10T09:26:33,541 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c837c3ac6e4d47bab78b5e6dfea348b0 2024-11-10T09:26:33,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/27e0d54bf31949dfa771cb547fe5fe56 is 50, key is test_row_0/B:col10/1731230791756/Put/seqid=0 2024-11-10T09:26:33,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742142_1318 (size=12301) 2024-11-10T09:26:33,558 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/27e0d54bf31949dfa771cb547fe5fe56 2024-11-10T09:26:33,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/dfc631904fb543679f588f4dd8740562 is 50, key is test_row_0/C:col10/1731230791756/Put/seqid=0 2024-11-10T09:26:33,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742143_1319 (size=12301) 2024-11-10T09:26:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:33,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. as already flushing 2024-11-10T09:26:33,909 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-10T09:26:33,909 INFO [Thread-1096 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1096. 2024-11-10T09:26:33,909 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:33,909 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,909 DEBUG [Thread-1096 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:33,909 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,909 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:33,909 DEBUG [Thread-1096 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:33,919 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-10T09:26:33,919 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-10T09:26:33,919 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:33,919 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:33,919 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,919 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,919 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,919 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:33,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:33,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:33,970 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/dfc631904fb543679f588f4dd8740562 2024-11-10T09:26:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/c837c3ac6e4d47bab78b5e6dfea348b0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c837c3ac6e4d47bab78b5e6dfea348b0 2024-11-10T09:26:33,986 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c837c3ac6e4d47bab78b5e6dfea348b0, entries=150, sequenceid=411, filesize=12.0 K 2024-11-10T09:26:33,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/27e0d54bf31949dfa771cb547fe5fe56 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/27e0d54bf31949dfa771cb547fe5fe56 2024-11-10T09:26:33,992 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/27e0d54bf31949dfa771cb547fe5fe56, entries=150, sequenceid=411, filesize=12.0 K 2024-11-10T09:26:33,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/dfc631904fb543679f588f4dd8740562 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dfc631904fb543679f588f4dd8740562 2024-11-10T09:26:33,997 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dfc631904fb543679f588f4dd8740562, entries=150, sequenceid=411, filesize=12.0 K 2024-11-10T09:26:33,998 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=26.84 KB/27480 for 799270abaa9e80dc47bbc7cec6771c00 in 867ms, sequenceid=411, compaction requested=false 2024-11-10T09:26:33,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 799270abaa9e80dc47bbc7cec6771c00: 2024-11-10T09:26:33,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:33,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-10T09:26:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-10T09:26:34,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T09:26:34,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9500 sec 2024-11-10T09:26:34,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.9540 sec 2024-11-10T09:26:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T09:26:34,197 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:40,520 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-11-10T09:26:40,520 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:26:40,520 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:40,520 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:40,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:40,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:26:40,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-10T09:26:40,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-10T09:26:40,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-10T09:26:40,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-10T09:26:40,521 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-10T09:26:40,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:26:40,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4307 2024-11-10T09:26:40,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4472 2024-11-10T09:26:40,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4198 2024-11-10T09:26:40,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4287 2024-11-10T09:26:40,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4470 2024-11-10T09:26:40,522 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:26:40,522 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:26:40,522 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:26:40,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:40,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:40,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:26:40,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:26:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T09:26:40,529 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230800529"}]},"ts":"1731230800529"} 2024-11-10T09:26:40,530 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:26:40,530 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:26:40,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:26:40,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, UNASSIGN}] 2024-11-10T09:26:40,532 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, UNASSIGN 2024-11-10T09:26:40,533 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=799270abaa9e80dc47bbc7cec6771c00, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:40,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, UNASSIGN because future has completed 2024-11-10T09:26:40,534 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:26:40,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:26:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T09:26:40,687 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:40,687 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:26:40,688 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 799270abaa9e80dc47bbc7cec6771c00, disabling compactions & flushes 2024-11-10T09:26:40,688 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:40,688 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:40,688 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. after waiting 0 ms 2024-11-10T09:26:40,688 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:40,688 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing 799270abaa9e80dc47bbc7cec6771c00 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T09:26:40,698 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/064858e4b45b46c4b3badcfcd6711a9d is 50, key is test_row_0/A:col10/1731230793917/Put/seqid=0 2024-11-10T09:26:40,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742144_1320 (size=12301) 2024-11-10T09:26:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T09:26:41,104 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/064858e4b45b46c4b3badcfcd6711a9d 2024-11-10T09:26:41,118 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/6665737eac794d44871db8bb069c4552 is 50, key is test_row_0/B:col10/1731230793917/Put/seqid=0 2024-11-10T09:26:41,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742145_1321 (size=12301) 2024-11-10T09:26:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T09:26:41,524 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/6665737eac794d44871db8bb069c4552 2024-11-10T09:26:41,537 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/ebb0ff2755be4c7da53b219393217842 is 50, key is test_row_0/C:col10/1731230793917/Put/seqid=0 2024-11-10T09:26:41,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742146_1322 (size=12301) 2024-11-10T09:26:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T09:26:41,943 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/ebb0ff2755be4c7da53b219393217842 2024-11-10T09:26:41,953 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/A/064858e4b45b46c4b3badcfcd6711a9d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/064858e4b45b46c4b3badcfcd6711a9d 2024-11-10T09:26:41,957 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/064858e4b45b46c4b3badcfcd6711a9d, entries=150, sequenceid=421, filesize=12.0 K 2024-11-10T09:26:41,957 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/B/6665737eac794d44871db8bb069c4552 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/6665737eac794d44871db8bb069c4552 2024-11-10T09:26:41,960 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/6665737eac794d44871db8bb069c4552, entries=150, sequenceid=421, filesize=12.0 K 2024-11-10T09:26:41,961 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/.tmp/C/ebb0ff2755be4c7da53b219393217842 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ebb0ff2755be4c7da53b219393217842 2024-11-10T09:26:41,964 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ebb0ff2755be4c7da53b219393217842, entries=150, sequenceid=421, filesize=12.0 K 2024-11-10T09:26:41,965 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 799270abaa9e80dc47bbc7cec6771c00 in 1277ms, sequenceid=421, compaction requested=true 2024-11-10T09:26:41,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/cb1e5ce642864c56951fc08ad8ebd181, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9bc934b949544d56981af7da4fe083fe, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ad42b06c741a4990ac9eaf759f28d88e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c6793d11ebc24dd5971bad35097122fd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/64201ba9cbe74d2f8a3f20673ee3ea1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/dfd89a8926814252bc023214946b3771, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/4d3bc033bec34a7b90535e283fb1f4c9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6b24ae31451440f2917032df53fc811a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/060dd9dee2614c688b8b49f837cd5ac5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/e1d702dd695341ec885b409890595c58, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/1e3eee8ab1804a75818699af31a3f09c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/fb5c1ec2813a47ea866e579497973564, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c45d5003ac284894b7be62ca24c81a97, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/06d008506cb04269a4a042e8b04a09c5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/47b07082b36947da94b3ba9cd37a34d0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/75abcc69f1124b37b66df89cd3003242, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c303415db548407e9ade0be9783321af, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9e8a64bd2e1e4999b1cbbc2acc74a028, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ef34f42a38914023afaa4c1bf276d78c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c7b333b615824ce08d6be453d1a979d3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3019aad774e74639824a4d7393e501fb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/f6379fb14d874e1c995ce3648a581c27, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3833f506d84649b4a1e8fb253f6d419f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/8a237ab1e2784e73aed08c39e50a03c7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6ec4a19ba26f4ce087dde9dcfad4d210, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/7c4fbdc493db4ce4b9f43d5ce5e55125] to archive 2024-11-10T09:26:41,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:26:41,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/cb1e5ce642864c56951fc08ad8ebd181 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/cb1e5ce642864c56951fc08ad8ebd181 2024-11-10T09:26:41,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/0edf2e2adcb94ba08cd9a2fdc2bfcdf9 2024-11-10T09:26:41,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9bc934b949544d56981af7da4fe083fe to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9bc934b949544d56981af7da4fe083fe 2024-11-10T09:26:41,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ad42b06c741a4990ac9eaf759f28d88e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ad42b06c741a4990ac9eaf759f28d88e 2024-11-10T09:26:41,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c6793d11ebc24dd5971bad35097122fd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c6793d11ebc24dd5971bad35097122fd 2024-11-10T09:26:41,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/64201ba9cbe74d2f8a3f20673ee3ea1d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/64201ba9cbe74d2f8a3f20673ee3ea1d 2024-11-10T09:26:41,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/dfd89a8926814252bc023214946b3771 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/dfd89a8926814252bc023214946b3771 2024-11-10T09:26:41,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/4d3bc033bec34a7b90535e283fb1f4c9 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/4d3bc033bec34a7b90535e283fb1f4c9 2024-11-10T09:26:41,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6b24ae31451440f2917032df53fc811a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6b24ae31451440f2917032df53fc811a 2024-11-10T09:26:41,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/060dd9dee2614c688b8b49f837cd5ac5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/060dd9dee2614c688b8b49f837cd5ac5 2024-11-10T09:26:41,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/e1d702dd695341ec885b409890595c58 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/e1d702dd695341ec885b409890595c58 2024-11-10T09:26:41,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/1e3eee8ab1804a75818699af31a3f09c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/1e3eee8ab1804a75818699af31a3f09c 2024-11-10T09:26:41,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/fb5c1ec2813a47ea866e579497973564 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/fb5c1ec2813a47ea866e579497973564 2024-11-10T09:26:41,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c45d5003ac284894b7be62ca24c81a97 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c45d5003ac284894b7be62ca24c81a97 2024-11-10T09:26:41,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/06d008506cb04269a4a042e8b04a09c5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/06d008506cb04269a4a042e8b04a09c5 2024-11-10T09:26:41,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/47b07082b36947da94b3ba9cd37a34d0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/47b07082b36947da94b3ba9cd37a34d0 2024-11-10T09:26:41,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/75abcc69f1124b37b66df89cd3003242 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/75abcc69f1124b37b66df89cd3003242 2024-11-10T09:26:41,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c303415db548407e9ade0be9783321af to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c303415db548407e9ade0be9783321af 2024-11-10T09:26:41,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9e8a64bd2e1e4999b1cbbc2acc74a028 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/9e8a64bd2e1e4999b1cbbc2acc74a028 2024-11-10T09:26:41,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ef34f42a38914023afaa4c1bf276d78c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/ef34f42a38914023afaa4c1bf276d78c 2024-11-10T09:26:41,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c7b333b615824ce08d6be453d1a979d3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c7b333b615824ce08d6be453d1a979d3 2024-11-10T09:26:41,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3019aad774e74639824a4d7393e501fb to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3019aad774e74639824a4d7393e501fb 2024-11-10T09:26:41,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/f6379fb14d874e1c995ce3648a581c27 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/f6379fb14d874e1c995ce3648a581c27 2024-11-10T09:26:41,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3833f506d84649b4a1e8fb253f6d419f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/3833f506d84649b4a1e8fb253f6d419f 2024-11-10T09:26:41,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/8a237ab1e2784e73aed08c39e50a03c7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/8a237ab1e2784e73aed08c39e50a03c7 2024-11-10T09:26:41,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6ec4a19ba26f4ce087dde9dcfad4d210 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/6ec4a19ba26f4ce087dde9dcfad4d210 2024-11-10T09:26:41,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/7c4fbdc493db4ce4b9f43d5ce5e55125 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/7c4fbdc493db4ce4b9f43d5ce5e55125 2024-11-10T09:26:41,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd056bb70eb9405988bc3e4f3b464d39, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3e2b26eb65004997bef4dc12c277bc8b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/7802943694af4530b50ee8fa51540806, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/c99d2234870242e9a1172d637cdd778a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/687c68e9061f48469842a5156aeb2495, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/9f5dbfe8b28f43c8aeb07882772a0d72, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/b507aa37518b42c5a67ec3888821fd43, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f3a66fecd4d84479b47fa8a866b366fc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/320e025ad0a94d26a4e6a1ebb6d4288d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ace483bb41fd4a8281f4c611780e4ebd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/532ebb8107a54267b3bba6f6412a2070, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/480ce0ad444d416886328b87f22f2ea4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d2852939575044059944114cdd007f74, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f7713f8ff05a45879100416e974cc6bf, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e0940e82acbc45c58e4efb18f71cfabf, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ed868a36b08a4196bea8fc65d6cce229, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd6a2e06f40a4e27b887cf53d6e1c461, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/46cd739687e2401299e3fd7b6f26bd4e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3d1d29bf7c0a4ce69a037ffb22844041, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3b76a2a6fa6b4bd9b978d15d1935603d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/510417a7f6524e35aef218603e2a43e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e2e20b4649ab4392a86a5b02f0d76b14, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/483e00d03a7a49bd8c3935802d180de1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/612a3808e2264a1bbe361bcb25293c1f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d36feed65b6f4221835055b01fab057a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/13cf2f1f6c564916942858982708fd75] to archive 2024-11-10T09:26:41,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:26:41,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd056bb70eb9405988bc3e4f3b464d39 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd056bb70eb9405988bc3e4f3b464d39 2024-11-10T09:26:41,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3e2b26eb65004997bef4dc12c277bc8b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3e2b26eb65004997bef4dc12c277bc8b 2024-11-10T09:26:42,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/7802943694af4530b50ee8fa51540806 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/7802943694af4530b50ee8fa51540806 2024-11-10T09:26:42,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/c99d2234870242e9a1172d637cdd778a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/c99d2234870242e9a1172d637cdd778a 2024-11-10T09:26:42,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/687c68e9061f48469842a5156aeb2495 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/687c68e9061f48469842a5156aeb2495 2024-11-10T09:26:42,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/9f5dbfe8b28f43c8aeb07882772a0d72 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/9f5dbfe8b28f43c8aeb07882772a0d72 2024-11-10T09:26:42,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/b507aa37518b42c5a67ec3888821fd43 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/b507aa37518b42c5a67ec3888821fd43 2024-11-10T09:26:42,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f3a66fecd4d84479b47fa8a866b366fc to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f3a66fecd4d84479b47fa8a866b366fc 2024-11-10T09:26:42,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/320e025ad0a94d26a4e6a1ebb6d4288d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/320e025ad0a94d26a4e6a1ebb6d4288d 2024-11-10T09:26:42,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ace483bb41fd4a8281f4c611780e4ebd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ace483bb41fd4a8281f4c611780e4ebd 2024-11-10T09:26:42,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/532ebb8107a54267b3bba6f6412a2070 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/532ebb8107a54267b3bba6f6412a2070 2024-11-10T09:26:42,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/480ce0ad444d416886328b87f22f2ea4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/480ce0ad444d416886328b87f22f2ea4 2024-11-10T09:26:42,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d2852939575044059944114cdd007f74 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d2852939575044059944114cdd007f74 2024-11-10T09:26:42,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/a3fb1fa0c1a44ae580c2b6c7ae8a9f09 2024-11-10T09:26:42,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f7713f8ff05a45879100416e974cc6bf to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/f7713f8ff05a45879100416e974cc6bf 2024-11-10T09:26:42,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e0940e82acbc45c58e4efb18f71cfabf to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e0940e82acbc45c58e4efb18f71cfabf 2024-11-10T09:26:42,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ed868a36b08a4196bea8fc65d6cce229 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/ed868a36b08a4196bea8fc65d6cce229 2024-11-10T09:26:42,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd6a2e06f40a4e27b887cf53d6e1c461 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/fd6a2e06f40a4e27b887cf53d6e1c461 2024-11-10T09:26:42,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/46cd739687e2401299e3fd7b6f26bd4e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/46cd739687e2401299e3fd7b6f26bd4e 2024-11-10T09:26:42,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3d1d29bf7c0a4ce69a037ffb22844041 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3d1d29bf7c0a4ce69a037ffb22844041 2024-11-10T09:26:42,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3b76a2a6fa6b4bd9b978d15d1935603d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/3b76a2a6fa6b4bd9b978d15d1935603d 2024-11-10T09:26:42,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/510417a7f6524e35aef218603e2a43e6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/510417a7f6524e35aef218603e2a43e6 2024-11-10T09:26:42,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e2e20b4649ab4392a86a5b02f0d76b14 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/e2e20b4649ab4392a86a5b02f0d76b14 2024-11-10T09:26:42,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/483e00d03a7a49bd8c3935802d180de1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/483e00d03a7a49bd8c3935802d180de1 2024-11-10T09:26:42,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/612a3808e2264a1bbe361bcb25293c1f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/612a3808e2264a1bbe361bcb25293c1f 2024-11-10T09:26:42,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d36feed65b6f4221835055b01fab057a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/d36feed65b6f4221835055b01fab057a 2024-11-10T09:26:42,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/13cf2f1f6c564916942858982708fd75 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/13cf2f1f6c564916942858982708fd75 2024-11-10T09:26:42,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f1bc68c6ade849aa83a5bf99dac8d8c8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9d10e6995b344fb299f4bffa79c8c624, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/22a7f46e282348ff91e8a721d95788fc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/bf91afc4e9e2461db44eb8106b8c27b6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dad5915db1ee497aadc7a6dde1a5a502, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/6cdeeabd39f84b5782fe6dbb8ab2705c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58407eab4f3541b3851345e88508ce1d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/2cf4e64ca1ab46ae838e2876c0719f50, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/11316933df674726b026d870473d3151, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/14a4e5413b664ee2a6efdb8901a08565, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a192dadc7898415eb77b4fc133941f04, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9c0539e123c1495d8d1d930a046895c7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/7f1c5655e4b145bd859cedaac281fae3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ef6fde3ad5ca4a4d8480c93aaecdbb92, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/262b9058ff044a78bfc360a7108f6367, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e9c43ce142f24882a5f98e0eab420396, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1d3b567d7d4947a9972c6c3ab503918d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f357182d114f4294a31dfa162be3e184, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a95ceeb5938e499caf2b252a012f2a00, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/cd34803864854069b22b2180d63215b1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1eb6112b4a194b71a01b847f0a2a3c60, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/4d128029a1d244ae8fbe2a0355f59f06, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/61939711084b41ea87a219fecf89f2cb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58f838de55aa462eb2790b93de646235, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/771df78f22c8447baa1d7f54b08d71f7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/5bd255903f7d4ed8b176aa54667f9a37, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/d9db1ce35a614a52ac24d89e352b4a1b] to archive 2024-11-10T09:26:42,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:26:42,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f1bc68c6ade849aa83a5bf99dac8d8c8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f1bc68c6ade849aa83a5bf99dac8d8c8 2024-11-10T09:26:42,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9d10e6995b344fb299f4bffa79c8c624 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9d10e6995b344fb299f4bffa79c8c624 2024-11-10T09:26:42,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/22a7f46e282348ff91e8a721d95788fc to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/22a7f46e282348ff91e8a721d95788fc 2024-11-10T09:26:42,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/bf91afc4e9e2461db44eb8106b8c27b6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/bf91afc4e9e2461db44eb8106b8c27b6 2024-11-10T09:26:42,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dad5915db1ee497aadc7a6dde1a5a502 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dad5915db1ee497aadc7a6dde1a5a502 2024-11-10T09:26:42,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/6cdeeabd39f84b5782fe6dbb8ab2705c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/6cdeeabd39f84b5782fe6dbb8ab2705c 2024-11-10T09:26:42,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58407eab4f3541b3851345e88508ce1d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58407eab4f3541b3851345e88508ce1d 2024-11-10T09:26:42,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/2cf4e64ca1ab46ae838e2876c0719f50 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/2cf4e64ca1ab46ae838e2876c0719f50 2024-11-10T09:26:42,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/11316933df674726b026d870473d3151 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/11316933df674726b026d870473d3151 2024-11-10T09:26:42,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/14a4e5413b664ee2a6efdb8901a08565 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/14a4e5413b664ee2a6efdb8901a08565 2024-11-10T09:26:42,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a192dadc7898415eb77b4fc133941f04 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a192dadc7898415eb77b4fc133941f04 2024-11-10T09:26:42,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9c0539e123c1495d8d1d930a046895c7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/9c0539e123c1495d8d1d930a046895c7 2024-11-10T09:26:42,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/7f1c5655e4b145bd859cedaac281fae3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/7f1c5655e4b145bd859cedaac281fae3 2024-11-10T09:26:42,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ef6fde3ad5ca4a4d8480c93aaecdbb92 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ef6fde3ad5ca4a4d8480c93aaecdbb92 2024-11-10T09:26:42,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/262b9058ff044a78bfc360a7108f6367 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/262b9058ff044a78bfc360a7108f6367 2024-11-10T09:26:42,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e9c43ce142f24882a5f98e0eab420396 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e9c43ce142f24882a5f98e0eab420396 2024-11-10T09:26:42,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1d3b567d7d4947a9972c6c3ab503918d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1d3b567d7d4947a9972c6c3ab503918d 2024-11-10T09:26:42,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f357182d114f4294a31dfa162be3e184 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/f357182d114f4294a31dfa162be3e184 2024-11-10T09:26:42,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a95ceeb5938e499caf2b252a012f2a00 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/a95ceeb5938e499caf2b252a012f2a00 2024-11-10T09:26:42,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/cd34803864854069b22b2180d63215b1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/cd34803864854069b22b2180d63215b1 2024-11-10T09:26:42,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1eb6112b4a194b71a01b847f0a2a3c60 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/1eb6112b4a194b71a01b847f0a2a3c60 2024-11-10T09:26:42,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/4d128029a1d244ae8fbe2a0355f59f06 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/4d128029a1d244ae8fbe2a0355f59f06 2024-11-10T09:26:42,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/61939711084b41ea87a219fecf89f2cb to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/61939711084b41ea87a219fecf89f2cb 2024-11-10T09:26:42,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58f838de55aa462eb2790b93de646235 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/58f838de55aa462eb2790b93de646235 2024-11-10T09:26:42,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/771df78f22c8447baa1d7f54b08d71f7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/771df78f22c8447baa1d7f54b08d71f7 2024-11-10T09:26:42,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/5bd255903f7d4ed8b176aa54667f9a37 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/5bd255903f7d4ed8b176aa54667f9a37 2024-11-10T09:26:42,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/d9db1ce35a614a52ac24d89e352b4a1b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/d9db1ce35a614a52ac24d89e352b4a1b 2024-11-10T09:26:42,081 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=1 2024-11-10T09:26:42,081 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. 2024-11-10T09:26:42,081 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 799270abaa9e80dc47bbc7cec6771c00: Waiting for close lock at 1731230800688Running coprocessor pre-close hooks at 1731230800688Disabling compacts and flushes for region at 1731230800688Disabling writes for close at 1731230800688Obtaining lock to block concurrent updates at 1731230800689 (+1 ms)Preparing flush snapshotting stores in 799270abaa9e80dc47bbc7cec6771c00 at 1731230800689Finished memstore snapshotting TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731230800689Flushing stores of TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00. at 1731230800691 (+2 ms)Flushing 799270abaa9e80dc47bbc7cec6771c00/A: creating writer at 1731230800691Flushing 799270abaa9e80dc47bbc7cec6771c00/A: appending metadata at 1731230800697 (+6 ms)Flushing 799270abaa9e80dc47bbc7cec6771c00/A: closing flushed file at 1731230800697Flushing 799270abaa9e80dc47bbc7cec6771c00/B: creating writer at 1731230801115 (+418 ms)Flushing 799270abaa9e80dc47bbc7cec6771c00/B: appending metadata at 1731230801118 (+3 ms)Flushing 799270abaa9e80dc47bbc7cec6771c00/B: closing flushed file at 1731230801118Flushing 799270abaa9e80dc47bbc7cec6771c00/C: creating writer at 1731230801533 (+415 ms)Flushing 799270abaa9e80dc47bbc7cec6771c00/C: appending metadata at 1731230801536 (+3 ms)Flushing 799270abaa9e80dc47bbc7cec6771c00/C: closing flushed file at 1731230801536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ed5689: reopening flushed file at 1731230801952 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48493406: reopening flushed file at 1731230801957 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73dbc758: reopening flushed file at 1731230801961 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 799270abaa9e80dc47bbc7cec6771c00 in 1277ms, sequenceid=421, compaction requested=true at 1731230801965 (+4 ms)Writing region close event to WAL at 1731230802077 (+112 ms)Running coprocessor post-close hooks at 1731230802081 (+4 ms)Closed at 1731230802081 2024-11-10T09:26:42,084 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:42,085 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=799270abaa9e80dc47bbc7cec6771c00, regionState=CLOSED 2024-11-10T09:26:42,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:26:42,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-10T09:26:42,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 799270abaa9e80dc47bbc7cec6771c00, server=802346a334f5,32883,1731230704436 in 1.5540 sec 2024-11-10T09:26:42,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-10T09:26:42,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=799270abaa9e80dc47bbc7cec6771c00, UNASSIGN in 1.5580 sec 2024-11-10T09:26:42,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T09:26:42,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5630 sec 2024-11-10T09:26:42,095 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230802095"}]},"ts":"1731230802095"} 2024-11-10T09:26:42,096 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:26:42,096 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:26:42,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5720 sec 2024-11-10T09:26:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T09:26:42,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:42,678 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:26:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:42,683 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-10T09:26:42,684 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:42,688 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:42,692 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/recovered.edits] 2024-11-10T09:26:42,697 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/064858e4b45b46c4b3badcfcd6711a9d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/064858e4b45b46c4b3badcfcd6711a9d 2024-11-10T09:26:42,698 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/12a15ea44e3e42ba99d44baee80b2a19 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/12a15ea44e3e42ba99d44baee80b2a19 2024-11-10T09:26:42,700 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c837c3ac6e4d47bab78b5e6dfea348b0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/A/c837c3ac6e4d47bab78b5e6dfea348b0 2024-11-10T09:26:42,704 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/27e0d54bf31949dfa771cb547fe5fe56 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/27e0d54bf31949dfa771cb547fe5fe56 2024-11-10T09:26:42,705 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/6665737eac794d44871db8bb069c4552 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/6665737eac794d44871db8bb069c4552 2024-11-10T09:26:42,707 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/85e257b48d9d4e8ebe510457b4253b2a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/B/85e257b48d9d4e8ebe510457b4253b2a 2024-11-10T09:26:42,711 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dfc631904fb543679f588f4dd8740562 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/dfc631904fb543679f588f4dd8740562 2024-11-10T09:26:42,712 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e0b16bb4d46a4d4b8f9812388fff7b6d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/e0b16bb4d46a4d4b8f9812388fff7b6d 2024-11-10T09:26:42,713 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ebb0ff2755be4c7da53b219393217842 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/C/ebb0ff2755be4c7da53b219393217842 2024-11-10T09:26:42,716 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/recovered.edits/424.seqid to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00/recovered.edits/424.seqid 2024-11-10T09:26:42,716 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/799270abaa9e80dc47bbc7cec6771c00 2024-11-10T09:26:42,716 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:26:42,718 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:42,721 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:26:42,723 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:26:42,724 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:42,724 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:26:42,724 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230802724"}]},"ts":"9223372036854775807"} 2024-11-10T09:26:42,725 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:26:42,725 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 799270abaa9e80dc47bbc7cec6771c00, NAME => 'TestAcidGuarantees,,1731230769904.799270abaa9e80dc47bbc7cec6771c00.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:26:42,725 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:26:42,726 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230802725"}]},"ts":"9223372036854775807"} 2024-11-10T09:26:42,727 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:26:42,728 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:42,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-10T09:26:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-10T09:26:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:26:42,796 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:42,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 241), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 459) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6879 (was 6959) 2024-11-10T09:26:42,816 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=6879 2024-11-10T09:26:42,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:26:42,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:26:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:42,819 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:26:42,820 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:42,820 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-10T09:26:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:26:42,821 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:26:42,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742147_1323 (size=959) 2024-11-10T09:26:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:26:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:26:43,231 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:26:43,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742148_1324 (size=53) 2024-11-10T09:26:43,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:26:43,642 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:26:43,642 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 82b4056cd8b4ce4bc1102670be5463a6, disabling compactions & flushes 2024-11-10T09:26:43,642 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:43,642 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:43,642 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. after waiting 0 ms 2024-11-10T09:26:43,642 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:43,642 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:43,643 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 82b4056cd8b4ce4bc1102670be5463a6: Waiting for close lock at 1731230803642Disabling compacts and flushes for region at 1731230803642Disabling writes for close at 1731230803642Writing region close event to WAL at 1731230803642Closed at 1731230803642 2024-11-10T09:26:43,646 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:26:43,647 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230803646"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230803646"}]},"ts":"1731230803646"} 2024-11-10T09:26:43,650 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:26:43,651 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:26:43,651 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230803651"}]},"ts":"1731230803651"} 2024-11-10T09:26:43,653 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:26:43,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, ASSIGN}] 2024-11-10T09:26:43,654 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, ASSIGN 2024-11-10T09:26:43,655 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, ASSIGN; state=OFFLINE, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=false 2024-11-10T09:26:43,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:26:43,806 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:43,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, ASSIGN because future has completed 2024-11-10T09:26:43,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:26:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:26:43,967 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:43,968 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:26:43,968 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,968 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:26:43,968 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,968 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,969 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,971 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b4056cd8b4ce4bc1102670be5463a6 columnFamilyName A 2024-11-10T09:26:43,971 DEBUG [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:43,971 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(327): Store=82b4056cd8b4ce4bc1102670be5463a6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:43,971 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,972 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b4056cd8b4ce4bc1102670be5463a6 columnFamilyName B 2024-11-10T09:26:43,972 DEBUG [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:43,973 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(327): Store=82b4056cd8b4ce4bc1102670be5463a6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:43,973 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,974 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b4056cd8b4ce4bc1102670be5463a6 columnFamilyName C 2024-11-10T09:26:43,974 DEBUG [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:43,974 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(327): Store=82b4056cd8b4ce4bc1102670be5463a6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:43,974 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,975 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,975 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,976 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,976 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,976 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:26:43,977 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,979 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:26:43,979 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 82b4056cd8b4ce4bc1102670be5463a6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72330210, jitterRate=0.07780411839485168}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:26:43,979 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:43,980 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 82b4056cd8b4ce4bc1102670be5463a6: Running coprocessor pre-open hook at 1731230803968Writing region info on filesystem at 1731230803968Initializing all the Stores at 1731230803969 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230803969Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230803969Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230803969Cleaning up temporary data from old regions at 1731230803976 (+7 ms)Running coprocessor post-open hooks at 1731230803979 (+3 ms)Region opened successfully at 1731230803980 (+1 ms) 2024-11-10T09:26:43,980 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., pid=99, masterSystemTime=1731230803965 2024-11-10T09:26:43,982 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:43,982 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:43,982 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:43,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:26:43,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-10T09:26:43,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 in 173 msec 2024-11-10T09:26:43,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T09:26:43,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, ASSIGN in 333 msec 2024-11-10T09:26:43,988 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:26:43,988 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230803988"}]},"ts":"1731230803988"} 2024-11-10T09:26:43,989 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:26:43,990 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:26:43,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-10T09:26:44,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T09:26:44,977 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:44,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59596ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:44,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:44,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:44,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a062d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:44,985 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261a2b11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:44,986 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:44,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:44,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:44,989 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:26:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,38235,1731230703274 2024-11-10T09:26:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1fea297d 2024-11-10T09:26:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:26:44,991 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:26:44,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:26:44,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:26:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T09:26:45,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742149_1325 (size=995) 2024-11-10T09:26:45,407 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T09:26:45,407 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T09:26:45,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:26:45,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, REOPEN/MOVE}] 2024-11-10T09:26:45,412 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, REOPEN/MOVE 2024-11-10T09:26:45,412 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:45,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, REOPEN/MOVE because future has completed 2024-11-10T09:26:45,414 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:26:45,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:26:45,566 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,566 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:26:45,566 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 82b4056cd8b4ce4bc1102670be5463a6, disabling compactions & flushes 2024-11-10T09:26:45,566 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:45,566 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:45,566 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. after waiting 0 ms 2024-11-10T09:26:45,566 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:45,574 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T09:26:45,575 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:45,575 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 82b4056cd8b4ce4bc1102670be5463a6: Waiting for close lock at 1731230805566Running coprocessor pre-close hooks at 1731230805566Disabling compacts and flushes for region at 1731230805566Disabling writes for close at 1731230805566Writing region close event to WAL at 1731230805567 (+1 ms)Running coprocessor post-close hooks at 1731230805575 (+8 ms)Closed at 1731230805575 2024-11-10T09:26:45,575 WARN [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 82b4056cd8b4ce4bc1102670be5463a6 to self. 2024-11-10T09:26:45,578 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,578 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=CLOSED 2024-11-10T09:26:45,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:26:45,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-10T09:26:45,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 in 167 msec 2024-11-10T09:26:45,583 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, REOPEN/MOVE; state=CLOSED, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=true 2024-11-10T09:26:45,734 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:45,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, REOPEN/MOVE because future has completed 2024-11-10T09:26:45,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:26:45,897 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:45,897 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:26:45,898 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,898 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:26:45,898 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,898 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,900 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,902 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b4056cd8b4ce4bc1102670be5463a6 columnFamilyName A 2024-11-10T09:26:45,904 DEBUG [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:45,904 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(327): Store=82b4056cd8b4ce4bc1102670be5463a6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:45,905 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,905 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b4056cd8b4ce4bc1102670be5463a6 columnFamilyName B 2024-11-10T09:26:45,905 DEBUG [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:45,906 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(327): Store=82b4056cd8b4ce4bc1102670be5463a6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:45,906 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,906 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82b4056cd8b4ce4bc1102670be5463a6 columnFamilyName C 2024-11-10T09:26:45,907 DEBUG [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:45,907 INFO [StoreOpener-82b4056cd8b4ce4bc1102670be5463a6-1 {}] regionserver.HStore(327): Store=82b4056cd8b4ce4bc1102670be5463a6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:26:45,907 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,908 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,908 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,909 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,909 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,910 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:26:45,910 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,911 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 82b4056cd8b4ce4bc1102670be5463a6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67256628, jitterRate=0.002201855182647705}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:26:45,911 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:45,911 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 82b4056cd8b4ce4bc1102670be5463a6: Running coprocessor pre-open hook at 1731230805898Writing region info on filesystem at 1731230805898Initializing all the Stores at 1731230805900 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230805900Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230805900Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230805900Cleaning up temporary data from old regions at 1731230805909 (+9 ms)Running coprocessor post-open hooks at 1731230805911 (+2 ms)Region opened successfully at 1731230805911 2024-11-10T09:26:45,912 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., pid=104, masterSystemTime=1731230805892 2024-11-10T09:26:45,914 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:45,914 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:45,914 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=OPEN, openSeqNum=5, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:26:45,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:26:45,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-10T09:26:45,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 in 179 msec 2024-11-10T09:26:45,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T09:26:45,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, REOPEN/MOVE in 506 msec 2024-11-10T09:26:45,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-10T09:26:45,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-10T09:26:45,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 928 msec 2024-11-10T09:26:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-10T09:26:45,923 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:45,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4853c07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,925 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126fc226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,926 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d041dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7d3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,928 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633fe209, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a28bd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b614aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e720a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,933 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df73d56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c87087, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,935 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d28b945, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,936 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fda646f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7679e723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,937 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648909bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,939 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f73383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f259004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@282ef3d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,942 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4050970b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526ed020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406692e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c73539c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2ec5f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4526396f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,947 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@433543ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ee78e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,949 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e99e41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,950 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c830385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ee060c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:26:45,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:26:45,952 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:26:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:26:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:26:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1112547d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:26:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:26:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:26:45,953 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:26:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c18f7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:26:45,956 DEBUG [Thread-1493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,956 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,956 DEBUG [Thread-1506 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,956 DEBUG [Thread-1512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,957 DEBUG [Thread-1501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,957 DEBUG [Thread-1506 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,958 DEBUG [Thread-1506 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,958 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T09:26:45,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,959 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:26:45,960 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:45,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,964 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:45,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,967 DEBUG [Thread-1504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,968 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,968 DEBUG [Thread-1508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:26:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:26:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:26:45,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:26:45,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:26:45,976 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:45,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:26:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:46,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104efe56bdb1a0482a889475a09be86714_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230805970/Put/seqid=0 2024-11-10T09:26:46,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742150_1326 (size=14594) 2024-11-10T09:26:46,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230866029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230866029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230866030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230866030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230866031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:26:46,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:46,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:46,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230866146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230866146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230866146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230866146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230866147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:26:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230866365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230866365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230866365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230866366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230866366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,422 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:46,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104efe56bdb1a0482a889475a09be86714_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104efe56bdb1a0482a889475a09be86714_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:46,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/46bc539db79940f290453b5c47a7b66d, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:46,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/46bc539db79940f290453b5c47a7b66d is 175, key is test_row_0/A:col10/1731230805970/Put/seqid=0 2024-11-10T09:26:46,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742151_1327 (size=39549) 2024-11-10T09:26:46,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:46,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:26:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230866676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230866676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230866676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230866676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230866685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:46,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:46,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:46,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/46bc539db79940f290453b5c47a7b66d 2024-11-10T09:26:46,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/075662f938c74fe59bfd8002c9191eb2 is 50, key is test_row_0/B:col10/1731230805970/Put/seqid=0 2024-11-10T09:26:46,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742152_1328 (size=12001) 2024-11-10T09:26:46,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:46,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:26:47,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230867186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230867186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230867195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230867195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230867195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:47,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/075662f938c74fe59bfd8002c9191eb2 2024-11-10T09:26:47,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/3839b31385244f07b56a232e86a72eb2 is 50, key is test_row_0/C:col10/1731230805970/Put/seqid=0 2024-11-10T09:26:47,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742153_1329 (size=12001) 2024-11-10T09:26:47,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,610 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:26:47,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:47,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/3839b31385244f07b56a232e86a72eb2 2024-11-10T09:26:47,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/46bc539db79940f290453b5c47a7b66d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/46bc539db79940f290453b5c47a7b66d 2024-11-10T09:26:47,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/46bc539db79940f290453b5c47a7b66d, entries=200, sequenceid=15, filesize=38.6 K 2024-11-10T09:26:47,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/075662f938c74fe59bfd8002c9191eb2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/075662f938c74fe59bfd8002c9191eb2 2024-11-10T09:26:47,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/075662f938c74fe59bfd8002c9191eb2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:26:47,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/3839b31385244f07b56a232e86a72eb2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/3839b31385244f07b56a232e86a72eb2 2024-11-10T09:26:47,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/3839b31385244f07b56a232e86a72eb2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T09:26:47,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 82b4056cd8b4ce4bc1102670be5463a6 in 1726ms, sequenceid=15, compaction requested=false 2024-11-10T09:26:47,702 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T09:26:47,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:47,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T09:26:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:47,798 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:26:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105711a88890f64c5e8938beb292d93220_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230806030/Put/seqid=0 2024-11-10T09:26:47,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742154_1330 (size=12154) 2024-11-10T09:26:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:26:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:48,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:48,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:48,212 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105711a88890f64c5e8938beb292d93220_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105711a88890f64c5e8938beb292d93220_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:48,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/259f1153328048a9b6502f2396ef29e6, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:48,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/259f1153328048a9b6502f2396ef29e6 is 175, key is test_row_0/A:col10/1731230806030/Put/seqid=0 2024-11-10T09:26:48,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742155_1331 (size=30955) 2024-11-10T09:26:48,216 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/259f1153328048a9b6502f2396ef29e6 2024-11-10T09:26:48,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230868213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230868216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230868216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230868218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,222 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 82b4056cd8b4ce4bc1102670be5463a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,222 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 82b4056cd8b4ce4bc1102670be5463a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/21861bf571bc41ef9e72835b849a5333 is 50, key is test_row_0/B:col10/1731230806030/Put/seqid=0 2024-11-10T09:26:48,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230868219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742156_1332 (size=12001) 2024-11-10T09:26:48,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230868326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230868326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230868335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230868536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230868546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230868546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,634 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/21861bf571bc41ef9e72835b849a5333 2024-11-10T09:26:48,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/e0c66d0f1803444880e821d1d4b2300f is 50, key is test_row_0/C:col10/1731230806030/Put/seqid=0 2024-11-10T09:26:48,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742157_1333 (size=12001) 2024-11-10T09:26:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230868845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230868855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230868855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:48,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:48,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,044 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/e0c66d0f1803444880e821d1d4b2300f 2024-11-10T09:26:49,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/259f1153328048a9b6502f2396ef29e6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/259f1153328048a9b6502f2396ef29e6 2024-11-10T09:26:49,058 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/259f1153328048a9b6502f2396ef29e6, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T09:26:49,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/21861bf571bc41ef9e72835b849a5333 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/21861bf571bc41ef9e72835b849a5333 2024-11-10T09:26:49,062 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/21861bf571bc41ef9e72835b849a5333, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T09:26:49,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/e0c66d0f1803444880e821d1d4b2300f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/e0c66d0f1803444880e821d1d4b2300f 2024-11-10T09:26:49,067 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/e0c66d0f1803444880e821d1d4b2300f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T09:26:49,068 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 82b4056cd8b4ce4bc1102670be5463a6 in 1270ms, sequenceid=40, compaction requested=false 2024-11-10T09:26:49,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:49,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:49,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T09:26:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T09:26:49,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T09:26:49,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1050 sec 2024-11-10T09:26:49,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.1150 sec 2024-11-10T09:26:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:49,359 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:49,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:26:49,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f4ae728a5b2a43608e6aaa3c380b95a7_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230809358/Put/seqid=0 2024-11-10T09:26:49,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742158_1334 (size=14594) 2024-11-10T09:26:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230869413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230869417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230869423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230869526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230869536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230869536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230869735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230869746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230869746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:49,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:49,773 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f4ae728a5b2a43608e6aaa3c380b95a7_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f4ae728a5b2a43608e6aaa3c380b95a7_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:49,773 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/49859f0039ec4810afd43e0656ed4993, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:49,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/49859f0039ec4810afd43e0656ed4993 is 175, key is test_row_0/A:col10/1731230809358/Put/seqid=0 2024-11-10T09:26:49,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742159_1335 (size=39549) 2024-11-10T09:26:50,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230870055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230870055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230870055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T09:26:50,136 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:50,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-10T09:26:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:26:50,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:50,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:50,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:50,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/49859f0039ec4810afd43e0656ed4993 2024-11-10T09:26:50,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/e5a3a8e4d17c468c91f82afca7b149c6 is 50, key is test_row_0/B:col10/1731230809358/Put/seqid=0 2024-11-10T09:26:50,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742160_1336 (size=12001) 2024-11-10T09:26:50,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230870235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,242 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 82b4056cd8b4ce4bc1102670be5463a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:26:50,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230870245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,251 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 82b4056cd8b4ce4bc1102670be5463a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T09:26:50,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:50,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T09:26:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:26:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230870565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230870565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230870565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:50,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/e5a3a8e4d17c468c91f82afca7b149c6 2024-11-10T09:26:50,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/53e90822d52b468e8e156740acdae229 is 50, key is test_row_0/C:col10/1731230809358/Put/seqid=0 2024-11-10T09:26:50,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742161_1337 (size=12001) 2024-11-10T09:26:50,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T09:26:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:50,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/53e90822d52b468e8e156740acdae229 2024-11-10T09:26:50,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:50,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/49859f0039ec4810afd43e0656ed4993 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/49859f0039ec4810afd43e0656ed4993 2024-11-10T09:26:50,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/49859f0039ec4810afd43e0656ed4993, entries=200, sequenceid=52, filesize=38.6 K 2024-11-10T09:26:50,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/e5a3a8e4d17c468c91f82afca7b149c6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/e5a3a8e4d17c468c91f82afca7b149c6 2024-11-10T09:26:50,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/e5a3a8e4d17c468c91f82afca7b149c6, entries=150, sequenceid=52, filesize=11.7 K 2024-11-10T09:26:50,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/53e90822d52b468e8e156740acdae229 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/53e90822d52b468e8e156740acdae229 2024-11-10T09:26:50,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/53e90822d52b468e8e156740acdae229, entries=150, sequenceid=52, filesize=11.7 K 2024-11-10T09:26:50,617 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T09:26:50,617 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T09:26:50,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 82b4056cd8b4ce4bc1102670be5463a6 in 1258ms, sequenceid=52, compaction requested=true 2024-11-10T09:26:50,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:50,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:50,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:50,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:50,618 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:50,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:50,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:50,618 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:50,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:50,619 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:50,619 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/A is initiating minor compaction (all files) 2024-11-10T09:26:50,619 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/A in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,619 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/46bc539db79940f290453b5c47a7b66d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/259f1153328048a9b6502f2396ef29e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/49859f0039ec4810afd43e0656ed4993] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=107.5 K 2024-11-10T09:26:50,619 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,619 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/46bc539db79940f290453b5c47a7b66d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/259f1153328048a9b6502f2396ef29e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/49859f0039ec4810afd43e0656ed4993] 2024-11-10T09:26:50,619 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46bc539db79940f290453b5c47a7b66d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230805968 2024-11-10T09:26:50,620 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 259f1153328048a9b6502f2396ef29e6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230806011 2024-11-10T09:26:50,620 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:50,620 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49859f0039ec4810afd43e0656ed4993, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731230808208 2024-11-10T09:26:50,620 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/B is initiating minor compaction (all files) 2024-11-10T09:26:50,620 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/B in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,620 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/075662f938c74fe59bfd8002c9191eb2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/21861bf571bc41ef9e72835b849a5333, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/e5a3a8e4d17c468c91f82afca7b149c6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=35.2 K 2024-11-10T09:26:50,620 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 075662f938c74fe59bfd8002c9191eb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230805968 2024-11-10T09:26:50,620 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 21861bf571bc41ef9e72835b849a5333, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230806011 2024-11-10T09:26:50,621 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a3a8e4d17c468c91f82afca7b149c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731230808208 2024-11-10T09:26:50,624 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:50,625 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:50,626 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/ce98538ea7e34e539f90cb45cae1c326 is 50, key is test_row_0/B:col10/1731230809358/Put/seqid=0 2024-11-10T09:26:50,626 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411101d6a66621d8f424892104e0cc672791b_82b4056cd8b4ce4bc1102670be5463a6 store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:50,628 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411101d6a66621d8f424892104e0cc672791b_82b4056cd8b4ce4bc1102670be5463a6, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:50,629 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101d6a66621d8f424892104e0cc672791b_82b4056cd8b4ce4bc1102670be5463a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:50,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742162_1338 (size=12104) 2024-11-10T09:26:50,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742163_1339 (size=4469) 2024-11-10T09:26:50,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T09:26:50,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:50,750 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:26:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110144d54a836ae45b7910b1216bc34b88d_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230809422/Put/seqid=0 2024-11-10T09:26:50,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742164_1340 (size=12154) 2024-11-10T09:26:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:26:51,033 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#A#compaction#291 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:51,034 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/66c14268a9174b849599b993a5305dbe is 175, key is test_row_0/A:col10/1731230809358/Put/seqid=0 2024-11-10T09:26:51,035 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/ce98538ea7e34e539f90cb45cae1c326 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/ce98538ea7e34e539f90cb45cae1c326 2024-11-10T09:26:51,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742165_1341 (size=31058) 2024-11-10T09:26:51,041 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/B of 82b4056cd8b4ce4bc1102670be5463a6 into ce98538ea7e34e539f90cb45cae1c326(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:51,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:51,041 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/B, priority=13, startTime=1731230810618; duration=0sec 2024-11-10T09:26:51,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:51,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:B 2024-11-10T09:26:51,041 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:51,044 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:51,044 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/C is initiating minor compaction (all files) 2024-11-10T09:26:51,044 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/C in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:51,044 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/3839b31385244f07b56a232e86a72eb2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/e0c66d0f1803444880e821d1d4b2300f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/53e90822d52b468e8e156740acdae229] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=35.2 K 2024-11-10T09:26:51,044 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3839b31385244f07b56a232e86a72eb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731230805968 2024-11-10T09:26:51,044 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e0c66d0f1803444880e821d1d4b2300f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230806011 2024-11-10T09:26:51,045 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e90822d52b468e8e156740acdae229, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731230808208 2024-11-10T09:26:51,053 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:51,053 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/41df090cf0b74226b98cb63ee8485961 is 50, key is test_row_0/C:col10/1731230809358/Put/seqid=0 2024-11-10T09:26:51,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742166_1342 (size=12104) 2024-11-10T09:26:51,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:51,162 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110144d54a836ae45b7910b1216bc34b88d_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110144d54a836ae45b7910b1216bc34b88d_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:51,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f86fe61643974f5094debcb13f907ab3, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:51,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f86fe61643974f5094debcb13f907ab3 is 175, key is test_row_0/A:col10/1731230809422/Put/seqid=0 2024-11-10T09:26:51,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742167_1343 (size=30955) 2024-11-10T09:26:51,169 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f86fe61643974f5094debcb13f907ab3 2024-11-10T09:26:51,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/b8ca6a33e0c04bad9477288ab063d550 is 50, key is test_row_0/B:col10/1731230809422/Put/seqid=0 2024-11-10T09:26:51,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742168_1344 (size=12001) 2024-11-10T09:26:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:26:51,441 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/66c14268a9174b849599b993a5305dbe as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/66c14268a9174b849599b993a5305dbe 2024-11-10T09:26:51,445 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/A of 82b4056cd8b4ce4bc1102670be5463a6 into 66c14268a9174b849599b993a5305dbe(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:51,445 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:51,445 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/A, priority=13, startTime=1731230810618; duration=0sec 2024-11-10T09:26:51,445 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:51,445 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:A 2024-11-10T09:26:51,462 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/41df090cf0b74226b98cb63ee8485961 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/41df090cf0b74226b98cb63ee8485961 2024-11-10T09:26:51,465 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/C of 82b4056cd8b4ce4bc1102670be5463a6 into 41df090cf0b74226b98cb63ee8485961(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:51,465 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:51,465 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/C, priority=13, startTime=1731230810618; duration=0sec 2024-11-10T09:26:51,465 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:51,465 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:C 2024-11-10T09:26:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:51,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:51,581 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/b8ca6a33e0c04bad9477288ab063d550 2024-11-10T09:26:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/6616b60b23d24ad8b1182eeb77aadfdc is 50, key is test_row_0/C:col10/1731230809422/Put/seqid=0 2024-11-10T09:26:51,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742169_1345 (size=12001) 2024-11-10T09:26:51,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230871598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230871602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230871603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230871705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230871715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230871715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230871915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230871925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230871926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,055 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/6616b60b23d24ad8b1182eeb77aadfdc 2024-11-10T09:26:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f86fe61643974f5094debcb13f907ab3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f86fe61643974f5094debcb13f907ab3 2024-11-10T09:26:52,062 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f86fe61643974f5094debcb13f907ab3, entries=150, sequenceid=76, filesize=30.2 K 2024-11-10T09:26:52,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/b8ca6a33e0c04bad9477288ab063d550 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/b8ca6a33e0c04bad9477288ab063d550 2024-11-10T09:26:52,065 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/b8ca6a33e0c04bad9477288ab063d550, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T09:26:52,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/6616b60b23d24ad8b1182eeb77aadfdc as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/6616b60b23d24ad8b1182eeb77aadfdc 2024-11-10T09:26:52,069 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/6616b60b23d24ad8b1182eeb77aadfdc, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T09:26:52,070 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 82b4056cd8b4ce4bc1102670be5463a6 in 1320ms, sequenceid=76, compaction requested=false 2024-11-10T09:26:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-10T09:26:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-10T09:26:52,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T09:26:52,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9330 sec 2024-11-10T09:26:52,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.9370 sec 2024-11-10T09:26:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:52,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:52,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T09:26:52,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110107a7c63803947d7a4bd8d4e97b02221_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230811593/Put/seqid=0 2024-11-10T09:26:52,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742170_1346 (size=14594) 2024-11-10T09:26:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T09:26:52,286 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:52,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-10T09:26:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:26:52,288 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:52,288 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:52,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230872281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230872281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230872282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:26:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230872396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230872396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230872405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:26:52,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230872616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230872616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230872616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,650 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:52,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110107a7c63803947d7a4bd8d4e97b02221_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110107a7c63803947d7a4bd8d4e97b02221_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:52,654 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f1d127c70be248f3ac9be0c95c9d5346, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:52,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f1d127c70be248f3ac9be0c95c9d5346 is 175, key is test_row_0/A:col10/1731230811593/Put/seqid=0 2024-11-10T09:26:52,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742171_1347 (size=39549) 2024-11-10T09:26:52,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:52,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:26:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230872925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230872936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230872936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:52,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:53,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f1d127c70be248f3ac9be0c95c9d5346 2024-11-10T09:26:53,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/3408d6862c424547887429dfa3e28469 is 50, key is test_row_0/B:col10/1731230811593/Put/seqid=0 2024-11-10T09:26:53,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742172_1348 (size=12001) 2024-11-10T09:26:53,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:26:53,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230873446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230873446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:53,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230873455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:53,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:53,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:53,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:53,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/3408d6862c424547887429dfa3e28469 2024-11-10T09:26:53,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/f928d28637704d7e81abe467358c1b2a is 50, key is test_row_0/C:col10/1731230811593/Put/seqid=0 2024-11-10T09:26:53,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742173_1349 (size=12001) 2024-11-10T09:26:53,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:26:53,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T09:26:53,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:53,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/f928d28637704d7e81abe467358c1b2a 2024-11-10T09:26:53,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/f1d127c70be248f3ac9be0c95c9d5346 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f1d127c70be248f3ac9be0c95c9d5346 2024-11-10T09:26:53,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f1d127c70be248f3ac9be0c95c9d5346, entries=200, sequenceid=93, filesize=38.6 K 2024-11-10T09:26:53,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/3408d6862c424547887429dfa3e28469 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/3408d6862c424547887429dfa3e28469 2024-11-10T09:26:53,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/3408d6862c424547887429dfa3e28469, entries=150, sequenceid=93, filesize=11.7 K 2024-11-10T09:26:53,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/f928d28637704d7e81abe467358c1b2a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/f928d28637704d7e81abe467358c1b2a 2024-11-10T09:26:53,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/f928d28637704d7e81abe467358c1b2a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-10T09:26:53,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 82b4056cd8b4ce4bc1102670be5463a6 in 1656ms, sequenceid=93, compaction requested=true 2024-11-10T09:26:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:53,893 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:53,893 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:53,894 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:53,894 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:53,894 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/A is initiating minor compaction (all files) 2024-11-10T09:26:53,894 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/B is initiating minor compaction (all files) 2024-11-10T09:26:53,894 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/A in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,894 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/B in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,894 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/66c14268a9174b849599b993a5305dbe, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f86fe61643974f5094debcb13f907ab3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f1d127c70be248f3ac9be0c95c9d5346] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=99.2 K 2024-11-10T09:26:53,894 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/ce98538ea7e34e539f90cb45cae1c326, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/b8ca6a33e0c04bad9477288ab063d550, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/3408d6862c424547887429dfa3e28469] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=35.3 K 2024-11-10T09:26:53,894 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,894 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/66c14268a9174b849599b993a5305dbe, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f86fe61643974f5094debcb13f907ab3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f1d127c70be248f3ac9be0c95c9d5346] 2024-11-10T09:26:53,894 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ce98538ea7e34e539f90cb45cae1c326, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731230808208 2024-11-10T09:26:53,894 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c14268a9174b849599b993a5305dbe, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731230808208 2024-11-10T09:26:53,895 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ca6a33e0c04bad9477288ab063d550, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230809395 2024-11-10T09:26:53,895 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86fe61643974f5094debcb13f907ab3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230809395 2024-11-10T09:26:53,895 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3408d6862c424547887429dfa3e28469, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731230811593 2024-11-10T09:26:53,895 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1d127c70be248f3ac9be0c95c9d5346, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731230811593 2024-11-10T09:26:53,899 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:53,900 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:53,901 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d1586ec2a1864934b50070d7e3c46b24_82b4056cd8b4ce4bc1102670be5463a6 store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:53,901 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/a2d84728cabe4510b5b1660333fcaf1a is 50, key is test_row_0/B:col10/1731230811593/Put/seqid=0 2024-11-10T09:26:53,903 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d1586ec2a1864934b50070d7e3c46b24_82b4056cd8b4ce4bc1102670be5463a6, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:53,903 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d1586ec2a1864934b50070d7e3c46b24_82b4056cd8b4ce4bc1102670be5463a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:53,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742175_1351 (size=4469) 2024-11-10T09:26:53,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742174_1350 (size=12207) 2024-11-10T09:26:53,913 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#A#compaction#300 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:53,913 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/ba79bb57d5c745e49236685f467631b4 is 175, key is test_row_0/A:col10/1731230811593/Put/seqid=0 2024-11-10T09:26:53,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742176_1352 (size=31161) 2024-11-10T09:26:53,921 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/ba79bb57d5c745e49236685f467631b4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/ba79bb57d5c745e49236685f467631b4 2024-11-10T09:26:53,925 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/A of 82b4056cd8b4ce4bc1102670be5463a6 into ba79bb57d5c745e49236685f467631b4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:53,925 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:53,925 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/A, priority=13, startTime=1731230813893; duration=0sec 2024-11-10T09:26:53,926 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:53,926 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:A 2024-11-10T09:26:53,926 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:53,926 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:53,926 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/C is initiating minor compaction (all files) 2024-11-10T09:26:53,926 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/C in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,926 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/41df090cf0b74226b98cb63ee8485961, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/6616b60b23d24ad8b1182eeb77aadfdc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/f928d28637704d7e81abe467358c1b2a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=35.3 K 2024-11-10T09:26:53,927 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41df090cf0b74226b98cb63ee8485961, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731230808208 2024-11-10T09:26:53,927 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6616b60b23d24ad8b1182eeb77aadfdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230809395 2024-11-10T09:26:53,927 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f928d28637704d7e81abe467358c1b2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731230811593 2024-11-10T09:26:53,931 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#C#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:53,932 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/034b48adfbd0496493f05d011cfaf5ba is 50, key is test_row_0/C:col10/1731230811593/Put/seqid=0 2024-11-10T09:26:53,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742177_1353 (size=12207) 2024-11-10T09:26:53,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-10T09:26:53,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:53,967 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-10T09:26:53,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111040eba69b437745caa89e37dd05129740_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230812269/Put/seqid=0 2024-11-10T09:26:53,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742178_1354 (size=12154) 2024-11-10T09:26:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:54,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230874300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230874304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,316 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/a2d84728cabe4510b5b1660333fcaf1a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/a2d84728cabe4510b5b1660333fcaf1a 2024-11-10T09:26:54,320 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/B of 82b4056cd8b4ce4bc1102670be5463a6 into a2d84728cabe4510b5b1660333fcaf1a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:54,320 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:54,320 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/B, priority=13, startTime=1731230813893; duration=0sec 2024-11-10T09:26:54,320 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:54,320 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:B 2024-11-10T09:26:54,339 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/034b48adfbd0496493f05d011cfaf5ba as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/034b48adfbd0496493f05d011cfaf5ba 2024-11-10T09:26:54,346 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/C of 82b4056cd8b4ce4bc1102670be5463a6 into 034b48adfbd0496493f05d011cfaf5ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:54,346 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:54,346 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/C, priority=13, startTime=1731230813893; duration=0sec 2024-11-10T09:26:54,346 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:54,346 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:C 2024-11-10T09:26:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:54,380 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111040eba69b437745caa89e37dd05129740_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111040eba69b437745caa89e37dd05129740_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/c9053939adab4c389f90f5e391bd1fec, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:54,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/c9053939adab4c389f90f5e391bd1fec is 175, key is test_row_0/A:col10/1731230812269/Put/seqid=0 2024-11-10T09:26:54,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742179_1355 (size=30955) 2024-11-10T09:26:54,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230874415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230874415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:26:54,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230874466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230874466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,471 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 82b4056cd8b4ce4bc1102670be5463a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:54,471 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 82b4056cd8b4ce4bc1102670be5463a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230874475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,481 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in 82b4056cd8b4ce4bc1102670be5463a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:26:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230874625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230874625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,786 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/c9053939adab4c389f90f5e391bd1fec 2024-11-10T09:26:54,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/39eebe2ee16d4935ad72862808e2985e is 50, key is test_row_0/B:col10/1731230812269/Put/seqid=0 2024-11-10T09:26:54,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742180_1356 (size=12001) 2024-11-10T09:26:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230874935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230874935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:55,201 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/39eebe2ee16d4935ad72862808e2985e 2024-11-10T09:26:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/8e979af3a7b54617ba62ec9f200d8526 is 50, key is test_row_0/C:col10/1731230812269/Put/seqid=0 2024-11-10T09:26:55,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742181_1357 (size=12001) 2024-11-10T09:26:55,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230875446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:55,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230875455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:55,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:55,612 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/8e979af3a7b54617ba62ec9f200d8526 2024-11-10T09:26:55,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/c9053939adab4c389f90f5e391bd1fec as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/c9053939adab4c389f90f5e391bd1fec 2024-11-10T09:26:55,619 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/c9053939adab4c389f90f5e391bd1fec, entries=150, sequenceid=116, filesize=30.2 K 2024-11-10T09:26:55,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/39eebe2ee16d4935ad72862808e2985e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/39eebe2ee16d4935ad72862808e2985e 2024-11-10T09:26:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,623 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/39eebe2ee16d4935ad72862808e2985e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T09:26:55,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/8e979af3a7b54617ba62ec9f200d8526 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/8e979af3a7b54617ba62ec9f200d8526 2024-11-10T09:26:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,631 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/8e979af3a7b54617ba62ec9f200d8526, entries=150, sequenceid=116, filesize=11.7 K 2024-11-10T09:26:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,632 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 82b4056cd8b4ce4bc1102670be5463a6 in 1664ms, sequenceid=116, compaction requested=false 2024-11-10T09:26:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-10T09:26:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-10T09:26:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T09:26:55,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3450 sec 2024-11-10T09:26:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.3490 sec 2024-11-10T09:26:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T09:26:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,467 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:26:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:26:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-10T09:26:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,469 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:26:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:26:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,470 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:26:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:26:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:56,485 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:56,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-10T09:26:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105da133d0f0884880bbf96c4c6a8e1c50_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230814303/Put/seqid=0 2024-11-10T09:26:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742182_1358 (size=12254) 2024-11-10T09:26:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105da133d0f0884880bbf96c4c6a8e1c50_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105da133d0f0884880bbf96c4c6a8e1c50_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/835f3e20e8984039bcce9bc954352ffc, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:56,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/835f3e20e8984039bcce9bc954352ffc is 175, key is test_row_0/A:col10/1731230814303/Put/seqid=0 2024-11-10T09:26:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742183_1359 (size=31051) 2024-11-10T09:26:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/835f3e20e8984039bcce9bc954352ffc 2024-11-10T09:26:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/f7f115ac725f42e69222d787d1978c65 is 50, key is test_row_0/B:col10/1731230814303/Put/seqid=0 2024-11-10T09:26:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742184_1360 (size=9757) 2024-11-10T09:26:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/f7f115ac725f42e69222d787d1978c65 2024-11-10T09:26:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/68c3418364e3423f811de68fa38587c6 is 50, key is test_row_0/C:col10/1731230814303/Put/seqid=0 2024-11-10T09:26:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742185_1361 (size=9757) 2024-11-10T09:26:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/68c3418364e3423f811de68fa38587c6 2024-11-10T09:26:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/835f3e20e8984039bcce9bc954352ffc as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/835f3e20e8984039bcce9bc954352ffc 2024-11-10T09:26:56,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/835f3e20e8984039bcce9bc954352ffc, entries=150, sequenceid=133, filesize=30.3 K 2024-11-10T09:26:56,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/f7f115ac725f42e69222d787d1978c65 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/f7f115ac725f42e69222d787d1978c65 2024-11-10T09:26:56,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230876561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:26:56,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/f7f115ac725f42e69222d787d1978c65, entries=100, sequenceid=133, filesize=9.5 K 2024-11-10T09:26:56,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/68c3418364e3423f811de68fa38587c6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/68c3418364e3423f811de68fa38587c6 2024-11-10T09:26:56,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/68c3418364e3423f811de68fa38587c6, entries=100, sequenceid=133, filesize=9.5 K 2024-11-10T09:26:56,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 82b4056cd8b4ce4bc1102670be5463a6 in 98ms, sequenceid=133, compaction requested=true 2024-11-10T09:26:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:26:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:56,583 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:26:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:56,583 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:26:56,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:56,587 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:56,587 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/A is initiating minor compaction (all files) 2024-11-10T09:26:56,587 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:56,587 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/A in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:26:56,588 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/ba79bb57d5c745e49236685f467631b4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/c9053939adab4c389f90f5e391bd1fec, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/835f3e20e8984039bcce9bc954352ffc] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=91.0 K 2024-11-10T09:26:56,588 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,588 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/ba79bb57d5c745e49236685f467631b4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/c9053939adab4c389f90f5e391bd1fec, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/835f3e20e8984039bcce9bc954352ffc] 2024-11-10T09:26:56,588 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:56,588 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/B is initiating minor compaction (all files) 2024-11-10T09:26:56,588 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/B in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,588 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/a2d84728cabe4510b5b1660333fcaf1a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/39eebe2ee16d4935ad72862808e2985e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/f7f115ac725f42e69222d787d1978c65] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=33.2 K 2024-11-10T09:26:56,590 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d84728cabe4510b5b1660333fcaf1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731230811593 2024-11-10T09:26:56,590 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba79bb57d5c745e49236685f467631b4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731230811593 2024-11-10T09:26:56,591 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9053939adab4c389f90f5e391bd1fec, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230812269 2024-11-10T09:26:56,591 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 39eebe2ee16d4935ad72862808e2985e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230812269 2024-11-10T09:26:56,592 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f115ac725f42e69222d787d1978c65, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230814303 2024-11-10T09:26:56,592 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835f3e20e8984039bcce9bc954352ffc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230814299 2024-11-10T09:26:56,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111024ef317305584f229d86fc0685fd372c_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230816567/Put/seqid=0 2024-11-10T09:26:56,603 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:56,604 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:56,604 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/4c914dd9045940299b96693c9d4c9dd9 is 50, key is test_row_0/B:col10/1731230814303/Put/seqid=0 2024-11-10T09:26:56,608 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111026ecda569abc443bafdb98e14233e42e_82b4056cd8b4ce4bc1102670be5463a6 store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:56,611 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111026ecda569abc443bafdb98e14233e42e_82b4056cd8b4ce4bc1102670be5463a6, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:56,611 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111026ecda569abc443bafdb98e14233e42e_82b4056cd8b4ce4bc1102670be5463a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742186_1362 (size=12409) 2024-11-10T09:26:56,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,627 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/4c914dd9045940299b96693c9d4c9dd9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/4c914dd9045940299b96693c9d4c9dd9 2024-11-10T09:26:56,632 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/B of 82b4056cd8b4ce4bc1102670be5463a6 into 4c914dd9045940299b96693c9d4c9dd9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:56,632 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:56,632 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/B, priority=13, startTime=1731230816583; duration=0sec 2024-11-10T09:26:56,632 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:26:56,632 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:B 2024-11-10T09:26:56,632 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:26:56,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230876622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,636 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:26:56,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,637 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/C is initiating minor compaction (all files) 2024-11-10T09:26:56,637 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/C in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230876624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230876625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,638 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/034b48adfbd0496493f05d011cfaf5ba, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/8e979af3a7b54617ba62ec9f200d8526, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/68c3418364e3423f811de68fa38587c6] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=33.2 K 2024-11-10T09:26:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230876626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,640 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 034b48adfbd0496493f05d011cfaf5ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731230811593 2024-11-10T09:26:56,640 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e979af3a7b54617ba62ec9f200d8526, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731230812269 2024-11-10T09:26:56,641 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c3418364e3423f811de68fa38587c6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230814303 2024-11-10T09:26:56,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742187_1363 (size=17284) 2024-11-10T09:26:56,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:56,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742188_1364 (size=4469) 2024-11-10T09:26:56,659 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#A#compaction#310 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:56,660 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/710bb03744f447c39cddee3f0033fec5 is 175, key is test_row_0/A:col10/1731230814303/Put/seqid=0 2024-11-10T09:26:56,661 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111024ef317305584f229d86fc0685fd372c_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111024ef317305584f229d86fc0685fd372c_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:56,664 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/b8fe61cb5523465c945481fd71b08a92, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:56,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/b8fe61cb5523465c945481fd71b08a92 is 175, key is test_row_0/A:col10/1731230816567/Put/seqid=0 2024-11-10T09:26:56,670 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:26:56,670 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/15f035844e764529bfa9a54007307ce7 is 50, key is test_row_0/C:col10/1731230814303/Put/seqid=0 2024-11-10T09:26:56,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742189_1365 (size=31470) 2024-11-10T09:26:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230876686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742190_1366 (size=48389) 2024-11-10T09:26:56,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/b8fe61cb5523465c945481fd71b08a92 2024-11-10T09:26:56,703 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/710bb03744f447c39cddee3f0033fec5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/710bb03744f447c39cddee3f0033fec5 2024-11-10T09:26:56,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/5fa34bbcb93b4525b07f8be39042ead1 is 50, key is test_row_0/B:col10/1731230816567/Put/seqid=0 2024-11-10T09:26:56,711 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/A of 82b4056cd8b4ce4bc1102670be5463a6 into 710bb03744f447c39cddee3f0033fec5(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:56,711 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:56,711 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/A, priority=13, startTime=1731230816583; duration=0sec 2024-11-10T09:26:56,712 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:56,712 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:A 2024-11-10T09:26:56,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742191_1367 (size=12409) 2024-11-10T09:26:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742192_1368 (size=12151) 2024-11-10T09:26:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230876745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230876745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230876745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230876746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:26:56,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230876905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:56,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:56,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230876955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230876956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230876956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230876956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:57,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:57,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:26:57,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/5fa34bbcb93b4525b07f8be39042ead1 2024-11-10T09:26:57,144 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/15f035844e764529bfa9a54007307ce7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/15f035844e764529bfa9a54007307ce7 2024-11-10T09:26:57,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/227cfe15e8204c4f9dd0b6c821863e4d is 50, key is test_row_0/C:col10/1731230816567/Put/seqid=0 2024-11-10T09:26:57,148 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/C of 82b4056cd8b4ce4bc1102670be5463a6 into 15f035844e764529bfa9a54007307ce7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:26:57,148 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:57,148 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/C, priority=13, startTime=1731230816583; duration=0sec 2024-11-10T09:26:57,148 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:26:57,149 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:C 2024-11-10T09:26:57,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742193_1369 (size=12151) 2024-11-10T09:26:57,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230877216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:57,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230877265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230877265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230877265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230877266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:57,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:57,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:57,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:57,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:26:57,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/227cfe15e8204c4f9dd0b6c821863e4d 2024-11-10T09:26:57,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/b8fe61cb5523465c945481fd71b08a92 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/b8fe61cb5523465c945481fd71b08a92 2024-11-10T09:26:57,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/b8fe61cb5523465c945481fd71b08a92, entries=250, sequenceid=157, filesize=47.3 K 2024-11-10T09:26:57,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/5fa34bbcb93b4525b07f8be39042ead1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/5fa34bbcb93b4525b07f8be39042ead1 2024-11-10T09:26:57,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/5fa34bbcb93b4525b07f8be39042ead1, entries=150, sequenceid=157, filesize=11.9 K 2024-11-10T09:26:57,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/227cfe15e8204c4f9dd0b6c821863e4d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/227cfe15e8204c4f9dd0b6c821863e4d 2024-11-10T09:26:57,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/227cfe15e8204c4f9dd0b6c821863e4d, entries=150, sequenceid=157, filesize=11.9 K 2024-11-10T09:26:57,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 82b4056cd8b4ce4bc1102670be5463a6 in 981ms, sequenceid=157, compaction requested=false 2024-11-10T09:26:57,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:26:57,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-10T09:26:57,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:57,692 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:26:57,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110448f437397c04ba0855730963183467b_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230816589/Put/seqid=0 2024-11-10T09:26:57,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742194_1370 (size=12304) 2024-11-10T09:26:57,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:26:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230877789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230877792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230877795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230877796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230877797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230877905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230877905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230877905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230877905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:57,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230877916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:26:58,103 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110448f437397c04ba0855730963183467b_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110448f437397c04ba0855730963183467b_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:58,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/76be074902eb4735a219d5286470e625, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:26:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/76be074902eb4735a219d5286470e625 is 175, key is test_row_0/A:col10/1731230816589/Put/seqid=0 2024-11-10T09:26:58,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742195_1371 (size=31105) 2024-11-10T09:26:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230878115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230878115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230878115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230878115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230878126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230878426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230878435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230878435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230878435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230878435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,507 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/76be074902eb4735a219d5286470e625 2024-11-10T09:26:58,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/6315a8e493dc4cce995b6ef53e3fe3fa is 50, key is test_row_0/B:col10/1731230816589/Put/seqid=0 2024-11-10T09:26:58,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742196_1372 (size=12151) 2024-11-10T09:26:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:26:58,918 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/6315a8e493dc4cce995b6ef53e3fe3fa 2024-11-10T09:26:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/fc9f387401144ffd8e670a0a0470cd04 is 50, key is test_row_0/C:col10/1731230816589/Put/seqid=0 2024-11-10T09:26:58,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742197_1373 (size=12151) 2024-11-10T09:26:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230878945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230878946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230878946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230878946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230878946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:59,327 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/fc9f387401144ffd8e670a0a0470cd04 2024-11-10T09:26:59,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/76be074902eb4735a219d5286470e625 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/76be074902eb4735a219d5286470e625 2024-11-10T09:26:59,334 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/76be074902eb4735a219d5286470e625, entries=150, sequenceid=173, filesize=30.4 K 2024-11-10T09:26:59,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/6315a8e493dc4cce995b6ef53e3fe3fa as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/6315a8e493dc4cce995b6ef53e3fe3fa 2024-11-10T09:26:59,337 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/6315a8e493dc4cce995b6ef53e3fe3fa, entries=150, sequenceid=173, filesize=11.9 K 2024-11-10T09:26:59,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/fc9f387401144ffd8e670a0a0470cd04 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/fc9f387401144ffd8e670a0a0470cd04 2024-11-10T09:26:59,340 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/fc9f387401144ffd8e670a0a0470cd04, entries=150, sequenceid=173, filesize=11.9 K 2024-11-10T09:26:59,341 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 82b4056cd8b4ce4bc1102670be5463a6 in 1649ms, sequenceid=173, compaction requested=true 2024-11-10T09:26:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:26:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:26:59,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-10T09:26:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-10T09:26:59,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T09:26:59,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8720 sec 2024-11-10T09:26:59,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.8770 sec 2024-11-10T09:26:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:26:59,960 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:26:59,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:26:59,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107a0b645681ec4dc1ab3d71648ae91152_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230819959/Put/seqid=0 2024-11-10T09:26:59,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742198_1374 (size=14794) 2024-11-10T09:26:59,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230879974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:59,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230879975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230879976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230879977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230879978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:26:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:26:59,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230880086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230880096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230880096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230880096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230880096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230880296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230880316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230880316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230880316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230880316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,370 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:00,373 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107a0b645681ec4dc1ab3d71648ae91152_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107a0b645681ec4dc1ab3d71648ae91152_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:00,374 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/a93028036f634a8f969c28b09d74ec88, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:00,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/a93028036f634a8f969c28b09d74ec88 is 175, key is test_row_0/A:col10/1731230819959/Put/seqid=0 2024-11-10T09:27:00,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742199_1375 (size=39749) 2024-11-10T09:27:00,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230880605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230880626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230880626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T09:27:00,636 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:00,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-10T09:27:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T09:27:00,638 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:00,639 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:00,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230880636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230880636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:00,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T09:27:00,777 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/a93028036f634a8f969c28b09d74ec88 2024-11-10T09:27:00,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/767e8b21b244462b8cbf2ab5857cb398 is 50, key is test_row_0/B:col10/1731230819959/Put/seqid=0 2024-11-10T09:27:00,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742200_1376 (size=12151) 2024-11-10T09:27:00,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/767e8b21b244462b8cbf2ab5857cb398 2024-11-10T09:27:00,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T09:27:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:27:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:00,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:00,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5c9439dd33264755964b1b0c4c1a9ee9 is 50, key is test_row_0/C:col10/1731230819959/Put/seqid=0 2024-11-10T09:27:00,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742201_1377 (size=12151) 2024-11-10T09:27:00,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T09:27:00,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:00,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:27:00,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:00,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:00,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T09:27:01,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T09:27:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:01,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:27:01,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:01,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:01,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:01,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230881116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:01,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230881135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:01,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230881145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:01,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230881156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230881156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:01,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5c9439dd33264755964b1b0c4c1a9ee9 2024-11-10T09:27:01,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/a93028036f634a8f969c28b09d74ec88 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/a93028036f634a8f969c28b09d74ec88 2024-11-10T09:27:01,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/a93028036f634a8f969c28b09d74ec88, entries=200, sequenceid=197, filesize=38.8 K 2024-11-10T09:27:01,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/767e8b21b244462b8cbf2ab5857cb398 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/767e8b21b244462b8cbf2ab5857cb398 2024-11-10T09:27:01,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/767e8b21b244462b8cbf2ab5857cb398, entries=150, sequenceid=197, filesize=11.9 K 2024-11-10T09:27:01,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5c9439dd33264755964b1b0c4c1a9ee9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5c9439dd33264755964b1b0c4c1a9ee9 2024-11-10T09:27:01,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5c9439dd33264755964b1b0c4c1a9ee9, entries=150, sequenceid=197, filesize=11.9 K 2024-11-10T09:27:01,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 82b4056cd8b4ce4bc1102670be5463a6 in 1252ms, sequenceid=197, compaction requested=true 2024-11-10T09:27:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:01,212 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:01,213 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:01,213 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150713 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:01,213 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:01,213 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/A is initiating minor compaction (all files) 2024-11-10T09:27:01,213 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/B is initiating minor compaction (all files) 2024-11-10T09:27:01,214 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/A in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:01,214 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/B in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:01,214 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/710bb03744f447c39cddee3f0033fec5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/b8fe61cb5523465c945481fd71b08a92, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/76be074902eb4735a219d5286470e625, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/a93028036f634a8f969c28b09d74ec88] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=147.2 K 2024-11-10T09:27:01,214 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/4c914dd9045940299b96693c9d4c9dd9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/5fa34bbcb93b4525b07f8be39042ead1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/6315a8e493dc4cce995b6ef53e3fe3fa, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/767e8b21b244462b8cbf2ab5857cb398] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=47.7 K 2024-11-10T09:27:01,214 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:01,214 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/710bb03744f447c39cddee3f0033fec5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/b8fe61cb5523465c945481fd71b08a92, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/76be074902eb4735a219d5286470e625, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/a93028036f634a8f969c28b09d74ec88] 2024-11-10T09:27:01,214 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c914dd9045940299b96693c9d4c9dd9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230812279 2024-11-10T09:27:01,214 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710bb03744f447c39cddee3f0033fec5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230812279 2024-11-10T09:27:01,214 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fa34bbcb93b4525b07f8be39042ead1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731230816567 2024-11-10T09:27:01,214 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8fe61cb5523465c945481fd71b08a92, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731230816567 2024-11-10T09:27:01,214 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 6315a8e493dc4cce995b6ef53e3fe3fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731230816589 2024-11-10T09:27:01,214 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76be074902eb4735a219d5286470e625, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731230816589 2024-11-10T09:27:01,215 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 767e8b21b244462b8cbf2ab5857cb398, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731230817794 2024-11-10T09:27:01,215 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93028036f634a8f969c28b09d74ec88, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731230817784 2024-11-10T09:27:01,222 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:01,223 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/deee21c75b41455690cfbc580fcc7ee7 is 50, key is test_row_0/B:col10/1731230819959/Put/seqid=0 2024-11-10T09:27:01,224 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:01,233 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110e3081ecaa28745de889c7dd80ec49c9f_82b4056cd8b4ce4bc1102670be5463a6 store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:01,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742202_1378 (size=12595) 2024-11-10T09:27:01,236 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110e3081ecaa28745de889c7dd80ec49c9f_82b4056cd8b4ce4bc1102670be5463a6, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:01,236 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e3081ecaa28745de889c7dd80ec49c9f_82b4056cd8b4ce4bc1102670be5463a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:01,238 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/deee21c75b41455690cfbc580fcc7ee7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/deee21c75b41455690cfbc580fcc7ee7 2024-11-10T09:27:01,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742203_1379 (size=4469) 2024-11-10T09:27:01,240 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#A#compaction#321 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:01,241 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/de9c5ae127e647f5959cdedd6f356747 is 175, key is test_row_0/A:col10/1731230819959/Put/seqid=0 2024-11-10T09:27:01,243 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/B of 82b4056cd8b4ce4bc1102670be5463a6 into deee21c75b41455690cfbc580fcc7ee7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:01,243 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:01,243 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/B, priority=12, startTime=1731230821212; duration=0sec 2024-11-10T09:27:01,243 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:01,243 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:B 2024-11-10T09:27:01,243 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:01,245 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:01,245 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/C is initiating minor compaction (all files) 2024-11-10T09:27:01,245 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/C in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:01,245 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/15f035844e764529bfa9a54007307ce7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/227cfe15e8204c4f9dd0b6c821863e4d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/fc9f387401144ffd8e670a0a0470cd04, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5c9439dd33264755964b1b0c4c1a9ee9] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=47.7 K 2024-11-10T09:27:01,246 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 15f035844e764529bfa9a54007307ce7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731230812279 2024-11-10T09:27:01,246 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 227cfe15e8204c4f9dd0b6c821863e4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731230816567 2024-11-10T09:27:01,246 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting fc9f387401144ffd8e670a0a0470cd04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731230816589 2024-11-10T09:27:01,247 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c9439dd33264755964b1b0c4c1a9ee9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731230817794 2024-11-10T09:27:01,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-10T09:27:01,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:01,249 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:27:01,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742204_1380 (size=31549) 2024-11-10T09:27:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d334e89584a84c1784d11ce03e78a2cd_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230819973/Put/seqid=0 2024-11-10T09:27:01,257 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:01,257 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/a8e06186d1ab4fc8b631b5eaa2f12cbb is 50, key is test_row_0/C:col10/1731230819959/Put/seqid=0 2024-11-10T09:27:01,261 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/de9c5ae127e647f5959cdedd6f356747 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/de9c5ae127e647f5959cdedd6f356747 2024-11-10T09:27:01,265 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/A of 82b4056cd8b4ce4bc1102670be5463a6 into de9c5ae127e647f5959cdedd6f356747(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:01,265 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:01,265 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/A, priority=12, startTime=1731230821212; duration=0sec 2024-11-10T09:27:01,266 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:01,266 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:A 2024-11-10T09:27:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T09:27:01,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742205_1381 (size=12304) 2024-11-10T09:27:01,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742206_1382 (size=12595) 2024-11-10T09:27:01,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:01,673 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/a8e06186d1ab4fc8b631b5eaa2f12cbb as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a8e06186d1ab4fc8b631b5eaa2f12cbb 2024-11-10T09:27:01,673 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d334e89584a84c1784d11ce03e78a2cd_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d334e89584a84c1784d11ce03e78a2cd_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:01,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/0ef40f9d23a14ba8b3507b0789d21a16, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:01,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/0ef40f9d23a14ba8b3507b0789d21a16 is 175, key is test_row_0/A:col10/1731230819973/Put/seqid=0 2024-11-10T09:27:01,678 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/C of 82b4056cd8b4ce4bc1102670be5463a6 into a8e06186d1ab4fc8b631b5eaa2f12cbb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:01,678 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:01,678 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/C, priority=12, startTime=1731230821213; duration=0sec 2024-11-10T09:27:01,678 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:01,678 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:C 2024-11-10T09:27:01,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742207_1383 (size=31105) 2024-11-10T09:27:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T09:27:02,080 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/0ef40f9d23a14ba8b3507b0789d21a16 2024-11-10T09:27:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:27:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/44d930731bf24034ae872956499b8a1a is 50, key is test_row_0/B:col10/1731230819973/Put/seqid=0 2024-11-10T09:27:02,146 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:27:02,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742208_1384 (size=12151) 2024-11-10T09:27:02,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230882182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230882183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230882184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230882189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230882190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230882296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230882296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230882296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230882296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230882305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230882506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230882506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230882506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230882506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230882515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,550 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/44d930731bf24034ae872956499b8a1a 2024-11-10T09:27:02,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/b7ff191179614d5dbfcdd01ea8188ea8 is 50, key is test_row_0/C:col10/1731230819973/Put/seqid=0 2024-11-10T09:27:02,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742209_1385 (size=12151) 2024-11-10T09:27:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T09:27:02,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230882816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230882816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230882816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230882825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230882825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:02,959 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/b7ff191179614d5dbfcdd01ea8188ea8 2024-11-10T09:27:02,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/0ef40f9d23a14ba8b3507b0789d21a16 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/0ef40f9d23a14ba8b3507b0789d21a16 2024-11-10T09:27:02,966 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/0ef40f9d23a14ba8b3507b0789d21a16, entries=150, sequenceid=210, filesize=30.4 K 2024-11-10T09:27:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/44d930731bf24034ae872956499b8a1a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/44d930731bf24034ae872956499b8a1a 2024-11-10T09:27:02,975 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/44d930731bf24034ae872956499b8a1a, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T09:27:02,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/b7ff191179614d5dbfcdd01ea8188ea8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/b7ff191179614d5dbfcdd01ea8188ea8 2024-11-10T09:27:02,979 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/b7ff191179614d5dbfcdd01ea8188ea8, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T09:27:02,979 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 82b4056cd8b4ce4bc1102670be5463a6 in 1730ms, sequenceid=210, compaction requested=false 2024-11-10T09:27:02,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-10T09:27:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-10T09:27:02,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T09:27:02,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3410 sec 2024-11-10T09:27:02,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.3450 sec 2024-11-10T09:27:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:03,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:03,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:27:03,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110605baba4470d431db7f455b739dd9622_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230822183/Put/seqid=0 2024-11-10T09:27:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742210_1386 (size=14794) 2024-11-10T09:27:03,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230883343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230883344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230883345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230883346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230883347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230883455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230883455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230883466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230883466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230883466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230883666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230883666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230883676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230883676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230883676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,742 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:03,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110605baba4470d431db7f455b739dd9622_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110605baba4470d431db7f455b739dd9622_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:03,746 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1ec846b9efff4e0fbf31faa4e215b963, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:03,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1ec846b9efff4e0fbf31faa4e215b963 is 175, key is test_row_0/A:col10/1731230822183/Put/seqid=0 2024-11-10T09:27:03,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742211_1387 (size=39749) 2024-11-10T09:27:03,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230883975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230883986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230883986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230883986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:04,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230883995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:04,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1ec846b9efff4e0fbf31faa4e215b963 2024-11-10T09:27:04,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/0d3e6aa195574253b35b3554246016da is 50, key is test_row_0/B:col10/1731230822183/Put/seqid=0 2024-11-10T09:27:04,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742212_1388 (size=12151) 2024-11-10T09:27:04,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230884496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:04,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230884496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230884505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230884505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230884505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:04,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/0d3e6aa195574253b35b3554246016da 2024-11-10T09:27:04,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5efec867bd8241498e985f5fcc0070ec is 50, key is test_row_0/C:col10/1731230822183/Put/seqid=0 2024-11-10T09:27:04,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742213_1389 (size=12151) 2024-11-10T09:27:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T09:27:04,816 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:04,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-10T09:27:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:27:04,818 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:04,818 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:04,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:27:04,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T09:27:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:27:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:04,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:04,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5efec867bd8241498e985f5fcc0070ec 2024-11-10T09:27:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:04,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1ec846b9efff4e0fbf31faa4e215b963 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1ec846b9efff4e0fbf31faa4e215b963 2024-11-10T09:27:04,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1ec846b9efff4e0fbf31faa4e215b963, entries=200, sequenceid=238, filesize=38.8 K 2024-11-10T09:27:04,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/0d3e6aa195574253b35b3554246016da as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/0d3e6aa195574253b35b3554246016da 2024-11-10T09:27:04,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/0d3e6aa195574253b35b3554246016da, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T09:27:04,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5efec867bd8241498e985f5fcc0070ec as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5efec867bd8241498e985f5fcc0070ec 2024-11-10T09:27:04,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5efec867bd8241498e985f5fcc0070ec, entries=150, sequenceid=238, filesize=11.9 K 2024-11-10T09:27:04,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 82b4056cd8b4ce4bc1102670be5463a6 in 1654ms, sequenceid=238, compaction requested=true 2024-11-10T09:27:04,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:04,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:04,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:04,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:04,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:04,984 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:04,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:04,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:04,984 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:04,985 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:04,985 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:04,985 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/A is initiating minor compaction (all files) 2024-11-10T09:27:04,985 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/B is initiating minor compaction (all files) 2024-11-10T09:27:04,985 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/A in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:04,985 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/B in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:04,985 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/de9c5ae127e647f5959cdedd6f356747, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/0ef40f9d23a14ba8b3507b0789d21a16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1ec846b9efff4e0fbf31faa4e215b963] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=100.0 K 2024-11-10T09:27:04,985 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/deee21c75b41455690cfbc580fcc7ee7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/44d930731bf24034ae872956499b8a1a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/0d3e6aa195574253b35b3554246016da] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=36.0 K 2024-11-10T09:27:04,985 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:04,985 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/de9c5ae127e647f5959cdedd6f356747, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/0ef40f9d23a14ba8b3507b0789d21a16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1ec846b9efff4e0fbf31faa4e215b963] 2024-11-10T09:27:04,986 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting deee21c75b41455690cfbc580fcc7ee7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731230817794 2024-11-10T09:27:04,986 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting de9c5ae127e647f5959cdedd6f356747, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731230817794 2024-11-10T09:27:04,986 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 44d930731bf24034ae872956499b8a1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230819973 2024-11-10T09:27:04,986 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ef40f9d23a14ba8b3507b0789d21a16, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230819973 2024-11-10T09:27:04,986 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3e6aa195574253b35b3554246016da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731230822183 2024-11-10T09:27:04,986 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec846b9efff4e0fbf31faa4e215b963, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731230822183 2024-11-10T09:27:04,991 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:04,992 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c083617efb7d4d2b936a915288cda26c_82b4056cd8b4ce4bc1102670be5463a6 store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:04,993 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:04,993 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/924988827b8147cfb4620715c992dd00 is 50, key is test_row_0/B:col10/1731230822183/Put/seqid=0 2024-11-10T09:27:04,994 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c083617efb7d4d2b936a915288cda26c_82b4056cd8b4ce4bc1102670be5463a6, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:04,994 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c083617efb7d4d2b936a915288cda26c_82b4056cd8b4ce4bc1102670be5463a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:05,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742214_1390 (size=12697) 2024-11-10T09:27:05,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742215_1391 (size=4469) 2024-11-10T09:27:05,011 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#A#compaction#329 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:05,012 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/2424188fbcac4985a0fdcb3fd2690272 is 175, key is test_row_0/A:col10/1731230822183/Put/seqid=0 2024-11-10T09:27:05,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742216_1392 (size=31651) 2024-11-10T09:27:05,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T09:27:05,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:05,123 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T09:27:05,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111074fd08d0ef204aff9e4e9604d1951256_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230823342/Put/seqid=0 2024-11-10T09:27:05,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742217_1393 (size=12304) 2024-11-10T09:27:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:27:05,136 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111074fd08d0ef204aff9e4e9604d1951256_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111074fd08d0ef204aff9e4e9604d1951256_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/3990983c8cd04215b5bd6f5ecd8c22b5, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:05,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/3990983c8cd04215b5bd6f5ecd8c22b5 is 175, key is test_row_0/A:col10/1731230823342/Put/seqid=0 2024-11-10T09:27:05,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742218_1394 (size=31105) 2024-11-10T09:27:05,412 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/924988827b8147cfb4620715c992dd00 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/924988827b8147cfb4620715c992dd00 2024-11-10T09:27:05,416 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/B of 82b4056cd8b4ce4bc1102670be5463a6 into 924988827b8147cfb4620715c992dd00(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:05,416 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:05,416 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/B, priority=13, startTime=1731230824984; duration=0sec 2024-11-10T09:27:05,416 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:05,416 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:B 2024-11-10T09:27:05,416 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:05,417 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:05,417 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 82b4056cd8b4ce4bc1102670be5463a6/C is initiating minor compaction (all files) 2024-11-10T09:27:05,417 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82b4056cd8b4ce4bc1102670be5463a6/C in TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:05,417 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a8e06186d1ab4fc8b631b5eaa2f12cbb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/b7ff191179614d5dbfcdd01ea8188ea8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5efec867bd8241498e985f5fcc0070ec] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp, totalSize=36.0 K 2024-11-10T09:27:05,417 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e06186d1ab4fc8b631b5eaa2f12cbb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731230817794 2024-11-10T09:27:05,417 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ff191179614d5dbfcdd01ea8188ea8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230819973 2024-11-10T09:27:05,418 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 5efec867bd8241498e985f5fcc0070ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731230822183 2024-11-10T09:27:05,419 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/2424188fbcac4985a0fdcb3fd2690272 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/2424188fbcac4985a0fdcb3fd2690272 2024-11-10T09:27:05,422 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/A of 82b4056cd8b4ce4bc1102670be5463a6 into 2424188fbcac4985a0fdcb3fd2690272(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:05,422 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:05,422 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/A, priority=13, startTime=1731230824984; duration=0sec 2024-11-10T09:27:05,422 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:05,422 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:A 2024-11-10T09:27:05,423 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82b4056cd8b4ce4bc1102670be5463a6#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:05,423 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/4e15e4631c684c10b1bca35037c070c6 is 50, key is test_row_0/C:col10/1731230822183/Put/seqid=0 2024-11-10T09:27:05,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742219_1395 (size=12697) 2024-11-10T09:27:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:27:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:05,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. as already flushing 2024-11-10T09:27:05,541 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/3990983c8cd04215b5bd6f5ecd8c22b5 2024-11-10T09:27:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/006d329c872c4324a79df3e6fbadfd76 is 50, key is test_row_0/B:col10/1731230823342/Put/seqid=0 2024-11-10T09:27:05,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742220_1396 (size=12151) 2024-11-10T09:27:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230885553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230885554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230885555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230885564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230885565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230885676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230885676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230885676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230885676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230885676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,837 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/4e15e4631c684c10b1bca35037c070c6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/4e15e4631c684c10b1bca35037c070c6 2024-11-10T09:27:05,841 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82b4056cd8b4ce4bc1102670be5463a6/C of 82b4056cd8b4ce4bc1102670be5463a6 into 4e15e4631c684c10b1bca35037c070c6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:05,841 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:05,841 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., storeName=82b4056cd8b4ce4bc1102670be5463a6/C, priority=13, startTime=1731230824984; duration=0sec 2024-11-10T09:27:05,841 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:05,841 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:C 2024-11-10T09:27:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230885886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230885886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230885886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230885886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230885886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:05,953 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/006d329c872c4324a79df3e6fbadfd76 2024-11-10T09:27:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:27:05,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5f5da7ed76b24534aa9d68400a769ab5 is 50, key is test_row_0/C:col10/1731230823342/Put/seqid=0 2024-11-10T09:27:05,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742221_1397 (size=12151) 2024-11-10T09:27:05,967 INFO [Thread-1504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1504. 2024-11-10T09:27:05,967 DEBUG [Thread-1504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:05,967 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,967 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:05,973 INFO [Thread-1506 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1506. 2024-11-10T09:27:05,973 DEBUG [Thread-1506 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:05,973 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,973 DEBUG [Thread-1506 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,973 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:05,974 INFO [Thread-1512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1512. 2024-11-10T09:27:05,974 DEBUG [Thread-1512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:05,974 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,974 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:05,975 INFO [Thread-1508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1508. 2024-11-10T09:27:05,975 DEBUG [Thread-1508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:05,975 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,975 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:05,980 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-11-10T09:27:05,980 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:05,980 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,980 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:05,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:06,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:06,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42190 deadline: 1731230886196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42216 deadline: 1731230886197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42222 deadline: 1731230886206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:06,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42236 deadline: 1731230886207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42252 deadline: 1731230886207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:06,364 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5f5da7ed76b24534aa9d68400a769ab5 2024-11-10T09:27:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/3990983c8cd04215b5bd6f5ecd8c22b5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/3990983c8cd04215b5bd6f5ecd8c22b5 2024-11-10T09:27:06,378 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/3990983c8cd04215b5bd6f5ecd8c22b5, entries=150, sequenceid=248, filesize=30.4 K 2024-11-10T09:27:06,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/006d329c872c4324a79df3e6fbadfd76 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/006d329c872c4324a79df3e6fbadfd76 2024-11-10T09:27:06,383 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/006d329c872c4324a79df3e6fbadfd76, entries=150, sequenceid=248, filesize=11.9 K 2024-11-10T09:27:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/5f5da7ed76b24534aa9d68400a769ab5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5f5da7ed76b24534aa9d68400a769ab5 2024-11-10T09:27:06,388 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5f5da7ed76b24534aa9d68400a769ab5, entries=150, sequenceid=248, filesize=11.9 K 2024-11-10T09:27:06,388 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 82b4056cd8b4ce4bc1102670be5463a6 in 1265ms, sequenceid=248, compaction requested=false 2024-11-10T09:27:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-10T09:27:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-10T09:27:06,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T09:27:06,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-11-10T09:27:06,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.5770 sec 2024-11-10T09:27:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:06,712 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:06,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-10T09:27:06,713 INFO [Thread-1493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1493. 2024-11-10T09:27:06,713 INFO [Thread-1501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1501. 2024-11-10T09:27:06,713 DEBUG [Thread-1493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:06,713 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,713 DEBUG [Thread-1501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:06,713 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,713 DEBUG [Thread-1493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:06,714 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:06,718 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-10T09:27:06,718 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-11-10T09:27:06,718 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:06,718 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:06,718 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,718 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,718 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,718 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:06,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:06,718 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-11-10T09:27:06,718 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:06,718 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,718 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:06,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f0d1c41a387b47bbb25357d6f3221625_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230826708/Put/seqid=0 2024-11-10T09:27:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742222_1398 (size=12454) 2024-11-10T09:27:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T09:27:06,967 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1518 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4554 rows 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1522 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4559 rows 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1522 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4566 rows 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4542 rows 2024-11-10T09:27:06,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1508 2024-11-10T09:27:06,969 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4521 rows 2024-11-10T09:27:06,969 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:27:06,969 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:27:06,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:06,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:06,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:27:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T09:27:06,977 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230826976"}]},"ts":"1731230826976"} 2024-11-10T09:27:06,978 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:27:06,978 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:27:06,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:27:06,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, UNASSIGN}] 2024-11-10T09:27:06,982 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, UNASSIGN 2024-11-10T09:27:06,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:06,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, UNASSIGN because future has completed 2024-11-10T09:27:06,985 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:27:06,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:27:07,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T09:27:07,124 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:07,137 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:07,137 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:27:07,138 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing 82b4056cd8b4ce4bc1102670be5463a6, disabling compactions & flushes 2024-11-10T09:27:07,138 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:07,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f0d1c41a387b47bbb25357d6f3221625_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f0d1c41a387b47bbb25357d6f3221625_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:07,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1874524efd964ae5b6379dd0a26f85de, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:07,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1874524efd964ae5b6379dd0a26f85de is 175, key is test_row_0/A:col10/1731230826708/Put/seqid=0 2024-11-10T09:27:07,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742223_1399 (size=31255) 2024-11-10T09:27:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T09:27:07,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/46bc539db79940f290453b5c47a7b66d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/259f1153328048a9b6502f2396ef29e6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/49859f0039ec4810afd43e0656ed4993, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/66c14268a9174b849599b993a5305dbe, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f86fe61643974f5094debcb13f907ab3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f1d127c70be248f3ac9be0c95c9d5346, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/ba79bb57d5c745e49236685f467631b4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/c9053939adab4c389f90f5e391bd1fec, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/710bb03744f447c39cddee3f0033fec5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/835f3e20e8984039bcce9bc954352ffc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/b8fe61cb5523465c945481fd71b08a92, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/76be074902eb4735a219d5286470e625, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/a93028036f634a8f969c28b09d74ec88, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/de9c5ae127e647f5959cdedd6f356747, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/0ef40f9d23a14ba8b3507b0789d21a16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1ec846b9efff4e0fbf31faa4e215b963] to archive 2024-11-10T09:27:07,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:27:07,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/46bc539db79940f290453b5c47a7b66d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/46bc539db79940f290453b5c47a7b66d 2024-11-10T09:27:07,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/259f1153328048a9b6502f2396ef29e6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/259f1153328048a9b6502f2396ef29e6 2024-11-10T09:27:07,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/49859f0039ec4810afd43e0656ed4993 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/49859f0039ec4810afd43e0656ed4993 2024-11-10T09:27:07,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/66c14268a9174b849599b993a5305dbe to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/66c14268a9174b849599b993a5305dbe 2024-11-10T09:27:07,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f86fe61643974f5094debcb13f907ab3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f86fe61643974f5094debcb13f907ab3 2024-11-10T09:27:07,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f1d127c70be248f3ac9be0c95c9d5346 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/f1d127c70be248f3ac9be0c95c9d5346 2024-11-10T09:27:07,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/ba79bb57d5c745e49236685f467631b4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/ba79bb57d5c745e49236685f467631b4 2024-11-10T09:27:07,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/c9053939adab4c389f90f5e391bd1fec to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/c9053939adab4c389f90f5e391bd1fec 2024-11-10T09:27:07,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/710bb03744f447c39cddee3f0033fec5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/710bb03744f447c39cddee3f0033fec5 2024-11-10T09:27:07,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/835f3e20e8984039bcce9bc954352ffc to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/835f3e20e8984039bcce9bc954352ffc 2024-11-10T09:27:07,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/b8fe61cb5523465c945481fd71b08a92 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/b8fe61cb5523465c945481fd71b08a92 2024-11-10T09:27:07,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/76be074902eb4735a219d5286470e625 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/76be074902eb4735a219d5286470e625 2024-11-10T09:27:07,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/a93028036f634a8f969c28b09d74ec88 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/a93028036f634a8f969c28b09d74ec88 2024-11-10T09:27:07,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/de9c5ae127e647f5959cdedd6f356747 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/de9c5ae127e647f5959cdedd6f356747 2024-11-10T09:27:07,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/0ef40f9d23a14ba8b3507b0789d21a16 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/0ef40f9d23a14ba8b3507b0789d21a16 2024-11-10T09:27:07,520 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1ec846b9efff4e0fbf31faa4e215b963 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1ec846b9efff4e0fbf31faa4e215b963 2024-11-10T09:27:07,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/075662f938c74fe59bfd8002c9191eb2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/21861bf571bc41ef9e72835b849a5333, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/ce98538ea7e34e539f90cb45cae1c326, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/e5a3a8e4d17c468c91f82afca7b149c6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/b8ca6a33e0c04bad9477288ab063d550, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/a2d84728cabe4510b5b1660333fcaf1a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/3408d6862c424547887429dfa3e28469, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/39eebe2ee16d4935ad72862808e2985e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/4c914dd9045940299b96693c9d4c9dd9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/f7f115ac725f42e69222d787d1978c65, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/5fa34bbcb93b4525b07f8be39042ead1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/6315a8e493dc4cce995b6ef53e3fe3fa, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/deee21c75b41455690cfbc580fcc7ee7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/767e8b21b244462b8cbf2ab5857cb398, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/44d930731bf24034ae872956499b8a1a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/0d3e6aa195574253b35b3554246016da] to archive 2024-11-10T09:27:07,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:27:07,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/075662f938c74fe59bfd8002c9191eb2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/075662f938c74fe59bfd8002c9191eb2 2024-11-10T09:27:07,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/21861bf571bc41ef9e72835b849a5333 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/21861bf571bc41ef9e72835b849a5333 2024-11-10T09:27:07,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/ce98538ea7e34e539f90cb45cae1c326 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/ce98538ea7e34e539f90cb45cae1c326 2024-11-10T09:27:07,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/e5a3a8e4d17c468c91f82afca7b149c6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/e5a3a8e4d17c468c91f82afca7b149c6 2024-11-10T09:27:07,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/b8ca6a33e0c04bad9477288ab063d550 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/b8ca6a33e0c04bad9477288ab063d550 2024-11-10T09:27:07,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/a2d84728cabe4510b5b1660333fcaf1a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/a2d84728cabe4510b5b1660333fcaf1a 2024-11-10T09:27:07,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/3408d6862c424547887429dfa3e28469 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/3408d6862c424547887429dfa3e28469 2024-11-10T09:27:07,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/39eebe2ee16d4935ad72862808e2985e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/39eebe2ee16d4935ad72862808e2985e 2024-11-10T09:27:07,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/4c914dd9045940299b96693c9d4c9dd9 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/4c914dd9045940299b96693c9d4c9dd9 2024-11-10T09:27:07,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/f7f115ac725f42e69222d787d1978c65 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/f7f115ac725f42e69222d787d1978c65 2024-11-10T09:27:07,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/5fa34bbcb93b4525b07f8be39042ead1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/5fa34bbcb93b4525b07f8be39042ead1 2024-11-10T09:27:07,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/6315a8e493dc4cce995b6ef53e3fe3fa to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/6315a8e493dc4cce995b6ef53e3fe3fa 2024-11-10T09:27:07,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/deee21c75b41455690cfbc580fcc7ee7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/deee21c75b41455690cfbc580fcc7ee7 2024-11-10T09:27:07,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/767e8b21b244462b8cbf2ab5857cb398 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/767e8b21b244462b8cbf2ab5857cb398 2024-11-10T09:27:07,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/44d930731bf24034ae872956499b8a1a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/44d930731bf24034ae872956499b8a1a 2024-11-10T09:27:07,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/0d3e6aa195574253b35b3554246016da to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/0d3e6aa195574253b35b3554246016da 2024-11-10T09:27:07,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/3839b31385244f07b56a232e86a72eb2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/e0c66d0f1803444880e821d1d4b2300f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/41df090cf0b74226b98cb63ee8485961, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/53e90822d52b468e8e156740acdae229, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/6616b60b23d24ad8b1182eeb77aadfdc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/034b48adfbd0496493f05d011cfaf5ba, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/f928d28637704d7e81abe467358c1b2a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/8e979af3a7b54617ba62ec9f200d8526, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/15f035844e764529bfa9a54007307ce7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/68c3418364e3423f811de68fa38587c6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/227cfe15e8204c4f9dd0b6c821863e4d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/fc9f387401144ffd8e670a0a0470cd04, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a8e06186d1ab4fc8b631b5eaa2f12cbb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5c9439dd33264755964b1b0c4c1a9ee9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/b7ff191179614d5dbfcdd01ea8188ea8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5efec867bd8241498e985f5fcc0070ec] to archive 2024-11-10T09:27:07,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:27:07,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/3839b31385244f07b56a232e86a72eb2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/3839b31385244f07b56a232e86a72eb2 2024-11-10T09:27:07,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/e0c66d0f1803444880e821d1d4b2300f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/e0c66d0f1803444880e821d1d4b2300f 2024-11-10T09:27:07,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/41df090cf0b74226b98cb63ee8485961 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/41df090cf0b74226b98cb63ee8485961 2024-11-10T09:27:07,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/53e90822d52b468e8e156740acdae229 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/53e90822d52b468e8e156740acdae229 2024-11-10T09:27:07,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/6616b60b23d24ad8b1182eeb77aadfdc to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/6616b60b23d24ad8b1182eeb77aadfdc 2024-11-10T09:27:07,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/034b48adfbd0496493f05d011cfaf5ba to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/034b48adfbd0496493f05d011cfaf5ba 2024-11-10T09:27:07,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/f928d28637704d7e81abe467358c1b2a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/f928d28637704d7e81abe467358c1b2a 2024-11-10T09:27:07,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/8e979af3a7b54617ba62ec9f200d8526 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/8e979af3a7b54617ba62ec9f200d8526 2024-11-10T09:27:07,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/15f035844e764529bfa9a54007307ce7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/15f035844e764529bfa9a54007307ce7 2024-11-10T09:27:07,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/68c3418364e3423f811de68fa38587c6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/68c3418364e3423f811de68fa38587c6 2024-11-10T09:27:07,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/227cfe15e8204c4f9dd0b6c821863e4d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/227cfe15e8204c4f9dd0b6c821863e4d 2024-11-10T09:27:07,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/fc9f387401144ffd8e670a0a0470cd04 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/fc9f387401144ffd8e670a0a0470cd04 2024-11-10T09:27:07,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a8e06186d1ab4fc8b631b5eaa2f12cbb to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a8e06186d1ab4fc8b631b5eaa2f12cbb 2024-11-10T09:27:07,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5c9439dd33264755964b1b0c4c1a9ee9 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5c9439dd33264755964b1b0c4c1a9ee9 2024-11-10T09:27:07,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/b7ff191179614d5dbfcdd01ea8188ea8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/b7ff191179614d5dbfcdd01ea8188ea8 2024-11-10T09:27:07,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5efec867bd8241498e985f5fcc0070ec to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5efec867bd8241498e985f5fcc0070ec 2024-11-10T09:27:07,595 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1874524efd964ae5b6379dd0a26f85de 2024-11-10T09:27:07,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/80fc2dcff32b420cb28fbe514c4a7329 is 50, key is test_row_0/B:col10/1731230826708/Put/seqid=0 2024-11-10T09:27:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T09:27:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742224_1400 (size=12301) 2024-11-10T09:27:08,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/80fc2dcff32b420cb28fbe514c4a7329 2024-11-10T09:27:08,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/a5552c1c394d4166943fe1fc9c8d9df4 is 50, key is test_row_0/C:col10/1731230826708/Put/seqid=0 2024-11-10T09:27:08,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742225_1401 (size=12301) 2024-11-10T09:27:08,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/a5552c1c394d4166943fe1fc9c8d9df4 2024-11-10T09:27:08,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/1874524efd964ae5b6379dd0a26f85de as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1874524efd964ae5b6379dd0a26f85de 2024-11-10T09:27:08,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1874524efd964ae5b6379dd0a26f85de, entries=150, sequenceid=280, filesize=30.5 K 2024-11-10T09:27:08,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/80fc2dcff32b420cb28fbe514c4a7329 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/80fc2dcff32b420cb28fbe514c4a7329 2024-11-10T09:27:08,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/80fc2dcff32b420cb28fbe514c4a7329, entries=150, sequenceid=280, filesize=12.0 K 2024-11-10T09:27:08,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/a5552c1c394d4166943fe1fc9c8d9df4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a5552c1c394d4166943fe1fc9c8d9df4 2024-11-10T09:27:08,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a5552c1c394d4166943fe1fc9c8d9df4, entries=150, sequenceid=280, filesize=12.0 K 2024-11-10T09:27:08,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=20.13 KB/20610 for 82b4056cd8b4ce4bc1102670be5463a6 in 1334ms, sequenceid=280, compaction requested=true 2024-11-10T09:27:08,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82b4056cd8b4ce4bc1102670be5463a6: 2024-11-10T09:27:08,046 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:08,046 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:08,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:08,046 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. because compaction request was cancelled 2024-11-10T09:27:08,046 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. after waiting 0 ms 2024-11-10T09:27:08,046 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:A 2024-11-10T09:27:08,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:08,046 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:08,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:B, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:08,046 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. because compaction request was cancelled 2024-11-10T09:27:08,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:08,046 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing 82b4056cd8b4ce4bc1102670be5463a6 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-10T09:27:08,046 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:B 2024-11-10T09:27:08,047 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. because compaction request was cancelled 2024-11-10T09:27:08,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82b4056cd8b4ce4bc1102670be5463a6:C, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:08,047 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82b4056cd8b4ce4bc1102670be5463a6:C 2024-11-10T09:27:08,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:08,051 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102f92f001d89042cfbd2b43ab226bfe99_82b4056cd8b4ce4bc1102670be5463a6 is 50, key is test_row_0/A:col10/1731230826716/Put/seqid=0 2024-11-10T09:27:08,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742226_1402 (size=9914) 2024-11-10T09:27:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T09:27:08,456 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:08,466 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102f92f001d89042cfbd2b43ab226bfe99_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102f92f001d89042cfbd2b43ab226bfe99_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:08,467 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/21e5cd70535842d3807a7801ec160e0e, store: [table=TestAcidGuarantees family=A region=82b4056cd8b4ce4bc1102670be5463a6] 2024-11-10T09:27:08,468 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/21e5cd70535842d3807a7801ec160e0e is 175, key is test_row_0/A:col10/1731230826716/Put/seqid=0 2024-11-10T09:27:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742227_1403 (size=22561) 2024-11-10T09:27:08,873 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/21e5cd70535842d3807a7801ec160e0e 2024-11-10T09:27:08,886 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/d87c35e473134ef9986d2af7e6da501d is 50, key is test_row_0/B:col10/1731230826716/Put/seqid=0 2024-11-10T09:27:08,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742228_1404 (size=9857) 2024-11-10T09:27:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T09:27:09,292 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/d87c35e473134ef9986d2af7e6da501d 2024-11-10T09:27:09,304 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/c087eb748e95412fa9c0e5b215013f13 is 50, key is test_row_0/C:col10/1731230826716/Put/seqid=0 2024-11-10T09:27:09,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742229_1405 (size=9857) 2024-11-10T09:27:09,709 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/c087eb748e95412fa9c0e5b215013f13 2024-11-10T09:27:09,719 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/A/21e5cd70535842d3807a7801ec160e0e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/21e5cd70535842d3807a7801ec160e0e 2024-11-10T09:27:09,725 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/21e5cd70535842d3807a7801ec160e0e, entries=100, sequenceid=286, filesize=22.0 K 2024-11-10T09:27:09,726 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/B/d87c35e473134ef9986d2af7e6da501d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/d87c35e473134ef9986d2af7e6da501d 2024-11-10T09:27:09,730 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/d87c35e473134ef9986d2af7e6da501d, entries=100, sequenceid=286, filesize=9.6 K 2024-11-10T09:27:09,731 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/.tmp/C/c087eb748e95412fa9c0e5b215013f13 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/c087eb748e95412fa9c0e5b215013f13 2024-11-10T09:27:09,734 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/c087eb748e95412fa9c0e5b215013f13, entries=100, sequenceid=286, filesize=9.6 K 2024-11-10T09:27:09,734 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 82b4056cd8b4ce4bc1102670be5463a6 in 1688ms, sequenceid=286, compaction requested=true 2024-11-10T09:27:09,738 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/recovered.edits/289.seqid, newMaxSeqId=289, maxSeqId=4 2024-11-10T09:27:09,738 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. 2024-11-10T09:27:09,738 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for 82b4056cd8b4ce4bc1102670be5463a6: Waiting for close lock at 1731230827137Running coprocessor pre-close hooks at 1731230827137Disabling compacts and flushes for region at 1731230827137Disabling writes for close at 1731230828046 (+909 ms)Obtaining lock to block concurrent updates at 1731230828046Preparing flush snapshotting stores in 82b4056cd8b4ce4bc1102670be5463a6 at 1731230828047 (+1 ms)Finished memstore snapshotting TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731230828047Flushing stores of TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6. at 1731230828047Flushing 82b4056cd8b4ce4bc1102670be5463a6/A: creating writer at 1731230828047Flushing mob file 82b4056cd8b4ce4bc1102670be5463a6/A: appending metadata at 1731230828051 (+4 ms)Flushing mob file 82b4056cd8b4ce4bc1102670be5463a6/A: closing flushed file at 1731230828051Flushing 82b4056cd8b4ce4bc1102670be5463a6/A: appending metadata at 1731230828467 (+416 ms)Flushing 82b4056cd8b4ce4bc1102670be5463a6/A: closing flushed file at 1731230828467Flushing 82b4056cd8b4ce4bc1102670be5463a6/B: creating writer at 1731230828881 (+414 ms)Flushing 82b4056cd8b4ce4bc1102670be5463a6/B: appending metadata at 1731230828885 (+4 ms)Flushing 82b4056cd8b4ce4bc1102670be5463a6/B: closing flushed file at 1731230828885Flushing 82b4056cd8b4ce4bc1102670be5463a6/C: creating writer at 1731230829300 (+415 ms)Flushing 82b4056cd8b4ce4bc1102670be5463a6/C: appending metadata at 1731230829304 (+4 ms)Flushing 82b4056cd8b4ce4bc1102670be5463a6/C: closing flushed file at 1731230829304Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19a5df10: reopening flushed file at 1731230829718 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65b8e0d3: reopening flushed file at 1731230829725 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33431e6a: reopening flushed file at 1731230829730 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 82b4056cd8b4ce4bc1102670be5463a6 in 1688ms, sequenceid=286, compaction requested=true at 1731230829734 (+4 ms)Writing region close event to WAL at 1731230829735 (+1 ms)Running coprocessor post-close hooks at 1731230829738 (+3 ms)Closed at 1731230829738 2024-11-10T09:27:09,740 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed 82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:09,740 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=82b4056cd8b4ce4bc1102670be5463a6, regionState=CLOSED 2024-11-10T09:27:09,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:27:09,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T09:27:09,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure 82b4056cd8b4ce4bc1102670be5463a6, server=802346a334f5,32883,1731230704436 in 2.7570 sec 2024-11-10T09:27:09,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-10T09:27:09,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82b4056cd8b4ce4bc1102670be5463a6, UNASSIGN in 2.7620 sec 2024-11-10T09:27:09,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T09:27:09,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7650 sec 2024-11-10T09:27:09,746 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230829746"}]},"ts":"1731230829746"} 2024-11-10T09:27:09,747 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:27:09,747 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:27:09,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7760 sec 2024-11-10T09:27:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T09:27:11,137 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:11,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:27:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:11,143 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:27:11,144 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:11,147 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,150 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/recovered.edits] 2024-11-10T09:27:11,155 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1874524efd964ae5b6379dd0a26f85de to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/1874524efd964ae5b6379dd0a26f85de 2024-11-10T09:27:11,157 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/21e5cd70535842d3807a7801ec160e0e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/21e5cd70535842d3807a7801ec160e0e 2024-11-10T09:27:11,159 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/2424188fbcac4985a0fdcb3fd2690272 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/2424188fbcac4985a0fdcb3fd2690272 2024-11-10T09:27:11,161 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/3990983c8cd04215b5bd6f5ecd8c22b5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/A/3990983c8cd04215b5bd6f5ecd8c22b5 2024-11-10T09:27:11,164 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/006d329c872c4324a79df3e6fbadfd76 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/006d329c872c4324a79df3e6fbadfd76 2024-11-10T09:27:11,165 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/80fc2dcff32b420cb28fbe514c4a7329 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/80fc2dcff32b420cb28fbe514c4a7329 2024-11-10T09:27:11,167 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/924988827b8147cfb4620715c992dd00 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/924988827b8147cfb4620715c992dd00 2024-11-10T09:27:11,168 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/d87c35e473134ef9986d2af7e6da501d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/B/d87c35e473134ef9986d2af7e6da501d 2024-11-10T09:27:11,171 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/4e15e4631c684c10b1bca35037c070c6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/4e15e4631c684c10b1bca35037c070c6 2024-11-10T09:27:11,172 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5f5da7ed76b24534aa9d68400a769ab5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/5f5da7ed76b24534aa9d68400a769ab5 2024-11-10T09:27:11,173 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a5552c1c394d4166943fe1fc9c8d9df4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/a5552c1c394d4166943fe1fc9c8d9df4 2024-11-10T09:27:11,175 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/c087eb748e95412fa9c0e5b215013f13 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/C/c087eb748e95412fa9c0e5b215013f13 2024-11-10T09:27:11,178 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/recovered.edits/289.seqid to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6/recovered.edits/289.seqid 2024-11-10T09:27:11,179 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,179 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:27:11,180 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:27:11,181 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T09:27:11,185 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110107a7c63803947d7a4bd8d4e97b02221_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110107a7c63803947d7a4bd8d4e97b02221_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,186 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110144d54a836ae45b7910b1216bc34b88d_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110144d54a836ae45b7910b1216bc34b88d_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,188 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111024ef317305584f229d86fc0685fd372c_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111024ef317305584f229d86fc0685fd372c_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,190 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102f92f001d89042cfbd2b43ab226bfe99_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102f92f001d89042cfbd2b43ab226bfe99_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,192 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111040eba69b437745caa89e37dd05129740_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111040eba69b437745caa89e37dd05129740_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,193 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110448f437397c04ba0855730963183467b_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110448f437397c04ba0855730963183467b_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,195 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104efe56bdb1a0482a889475a09be86714_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104efe56bdb1a0482a889475a09be86714_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,196 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105711a88890f64c5e8938beb292d93220_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105711a88890f64c5e8938beb292d93220_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,197 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105da133d0f0884880bbf96c4c6a8e1c50_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105da133d0f0884880bbf96c4c6a8e1c50_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,198 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110605baba4470d431db7f455b739dd9622_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110605baba4470d431db7f455b739dd9622_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,199 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111074fd08d0ef204aff9e4e9604d1951256_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111074fd08d0ef204aff9e4e9604d1951256_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,200 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107a0b645681ec4dc1ab3d71648ae91152_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107a0b645681ec4dc1ab3d71648ae91152_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,202 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d334e89584a84c1784d11ce03e78a2cd_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d334e89584a84c1784d11ce03e78a2cd_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,203 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f0d1c41a387b47bbb25357d6f3221625_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f0d1c41a387b47bbb25357d6f3221625_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,204 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f4ae728a5b2a43608e6aaa3c380b95a7_82b4056cd8b4ce4bc1102670be5463a6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f4ae728a5b2a43608e6aaa3c380b95a7_82b4056cd8b4ce4bc1102670be5463a6 2024-11-10T09:27:11,204 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:27:11,206 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:11,209 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:27:11,211 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:27:11,212 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:11,212 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:27:11,212 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230831212"}]},"ts":"9223372036854775807"} 2024-11-10T09:27:11,214 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:27:11,214 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 82b4056cd8b4ce4bc1102670be5463a6, NAME => 'TestAcidGuarantees,,1731230802817.82b4056cd8b4ce4bc1102670be5463a6.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:27:11,214 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:27:11,214 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230831214"}]},"ts":"9223372036854775807"} 2024-11-10T09:27:11,216 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:27:11,217 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:11,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-10T09:27:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T09:27:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:27:11,246 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:11,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=243 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1008509712_22 at /127.0.0.1:43872 [Waiting for operation #1121] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/802346a334f5:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1990253602_22 at /127.0.0.1:46816 [Waiting for operation #1312] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1990253602_22 at /127.0.0.1:43858 [Waiting for operation #1140] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1008509712_22 at /127.0.0.1:43854 [Waiting for operation #1142] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=445 (was 481), ProcessCount=11 (was 11), AvailableMemoryMB=6850 (was 6879) 2024-11-10T09:27:11,265 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=243, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=445, ProcessCount=11, AvailableMemoryMB=6850 2024-11-10T09:27:11,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:27:11,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:27:11,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:11,268 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:27:11,268 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:11,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-10T09:27:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-10T09:27:11,269 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:27:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742230_1406 (size=959) 2024-11-10T09:27:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-10T09:27:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-10T09:27:11,679 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:27:11,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742231_1407 (size=53) 2024-11-10T09:27:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-10T09:27:12,089 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:27:12,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 41bd7632cda810f44aae6fb0260d0e2d, disabling compactions & flushes 2024-11-10T09:27:12,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:12,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:12,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. after waiting 0 ms 2024-11-10T09:27:12,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:12,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:12,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 41bd7632cda810f44aae6fb0260d0e2d: Waiting for close lock at 1731230832090Disabling compacts and flushes for region at 1731230832090Disabling writes for close at 1731230832090Writing region close event to WAL at 1731230832090Closed at 1731230832090 2024-11-10T09:27:12,092 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:27:12,092 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230832092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230832092"}]},"ts":"1731230832092"} 2024-11-10T09:27:12,094 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:27:12,095 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:27:12,095 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230832095"}]},"ts":"1731230832095"} 2024-11-10T09:27:12,097 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:27:12,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, ASSIGN}] 2024-11-10T09:27:12,099 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, ASSIGN 2024-11-10T09:27:12,100 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, ASSIGN; state=OFFLINE, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=false 2024-11-10T09:27:12,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=41bd7632cda810f44aae6fb0260d0e2d, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:12,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, ASSIGN because future has completed 2024-11-10T09:27:12,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:27:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-10T09:27:12,479 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:12,479 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:27:12,481 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,481 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:27:12,482 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,482 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,484 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,486 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41bd7632cda810f44aae6fb0260d0e2d columnFamilyName A 2024-11-10T09:27:12,486 DEBUG [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:12,486 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] regionserver.HStore(327): Store=41bd7632cda810f44aae6fb0260d0e2d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:12,486 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,488 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41bd7632cda810f44aae6fb0260d0e2d columnFamilyName B 2024-11-10T09:27:12,488 DEBUG [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:12,488 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] regionserver.HStore(327): Store=41bd7632cda810f44aae6fb0260d0e2d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:12,488 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,490 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41bd7632cda810f44aae6fb0260d0e2d columnFamilyName C 2024-11-10T09:27:12,490 DEBUG [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:12,490 INFO [StoreOpener-41bd7632cda810f44aae6fb0260d0e2d-1 {}] regionserver.HStore(327): Store=41bd7632cda810f44aae6fb0260d0e2d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:12,490 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,491 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,491 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,493 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,493 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,493 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:27:12,495 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,497 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:27:12,497 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 41bd7632cda810f44aae6fb0260d0e2d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71277875, jitterRate=0.06212310492992401}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:27:12,497 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:12,498 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 41bd7632cda810f44aae6fb0260d0e2d: Running coprocessor pre-open hook at 1731230832482Writing region info on filesystem at 1731230832482Initializing all the Stores at 1731230832483 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230832483Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230832484 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230832484Cleaning up temporary data from old regions at 1731230832493 (+9 ms)Running coprocessor post-open hooks at 1731230832497 (+4 ms)Region opened successfully at 1731230832498 (+1 ms) 2024-11-10T09:27:12,499 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., pid=124, masterSystemTime=1731230832471 2024-11-10T09:27:12,502 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:12,502 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:12,503 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=41bd7632cda810f44aae6fb0260d0e2d, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:12,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:27:12,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T09:27:12,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 in 251 msec 2024-11-10T09:27:12,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-10T09:27:12,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, ASSIGN in 412 msec 2024-11-10T09:27:12,513 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:27:12,514 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230832513"}]},"ts":"1731230832513"} 2024-11-10T09:27:12,516 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:27:12,517 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:27:12,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2510 sec 2024-11-10T09:27:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-10T09:27:13,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:13,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5524d889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,423 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f62ed56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12db0ee5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,426 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:27:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,38235,1731230703274 2024-11-10T09:27:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b103cf 2024-11-10T09:27:13,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:27:13,430 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:27:13,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc683cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,432 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a88d2c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,433 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ec562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43681357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,435 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600c7818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,436 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f9a716, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39982088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,437 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371331dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,439 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2a8c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4c85de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f52de5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,442 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e67ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38404fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,444 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b02bfbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,445 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c59a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@125833c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,446 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,447 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432231fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,448 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4118a373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea16345, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,450 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7628e74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,451 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2116348b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2334b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,453 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de36b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b22638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1af8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,455 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76cce33f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,456 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a78429, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c40e836, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:13,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:13,458 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d5188b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:13,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:13,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:13,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a3a0e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:13,460 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,460 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,460 DEBUG [Thread-1856 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,460 DEBUG [Thread-1850 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,460 DEBUG [Thread-1854 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,460 DEBUG [Thread-1858 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,461 DEBUG [Thread-1867 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,461 DEBUG [Thread-1865 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,461 DEBUG [Thread-1865 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:13,462 DEBUG [Thread-1856 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,462 DEBUG [Thread-1856 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,463 DEBUG [Thread-1852 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,463 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,463 DEBUG [Thread-1869 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-10T09:27:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:13,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:27:13,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:13,468 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:13,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2] 2024-11-10T09:27:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:13,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:13,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:27:13,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/85d4a46f607b492692113b2d72c0ea10 is 50, key is test_row_0/A:col10/1731230833469/Put/seqid=0 2024-11-10T09:27:13,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742232_1408 (size=12001) 2024-11-10T09:27:13,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230893502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230893503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230893511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230893511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230893512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:27:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230893615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230893615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:13,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230893626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230893626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230893626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:13,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:27:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230893825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230893825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230893836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230893836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230893836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:13,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:13,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/85d4a46f607b492692113b2d72c0ea10 2024-11-10T09:27:13,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ad37bbf4f1a242ff913bc60659a80f98 is 50, key is test_row_0/B:col10/1731230833469/Put/seqid=0 2024-11-10T09:27:13,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:13,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:13,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:13,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:13,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:13,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742233_1409 (size=12001) 2024-11-10T09:27:14,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:14,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:14,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:27:14,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230894135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230894135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230894146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230894146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230894146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:14,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:14,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ad37bbf4f1a242ff913bc60659a80f98 2024-11-10T09:27:14,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/f7352ec59d1f450e9a60746d646c44a3 is 50, key is test_row_0/C:col10/1731230833469/Put/seqid=0 2024-11-10T09:27:14,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742234_1410 (size=12001) 2024-11-10T09:27:14,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:14,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:14,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:14,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:14,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:27:14,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230894646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230894646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230894656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230894656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230894665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:14,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:14,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:14,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/f7352ec59d1f450e9a60746d646c44a3 2024-11-10T09:27:14,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/85d4a46f607b492692113b2d72c0ea10 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/85d4a46f607b492692113b2d72c0ea10 2024-11-10T09:27:14,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/85d4a46f607b492692113b2d72c0ea10, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T09:27:14,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ad37bbf4f1a242ff913bc60659a80f98 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ad37bbf4f1a242ff913bc60659a80f98 2024-11-10T09:27:14,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ad37bbf4f1a242ff913bc60659a80f98, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T09:27:14,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/f7352ec59d1f450e9a60746d646c44a3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/f7352ec59d1f450e9a60746d646c44a3 2024-11-10T09:27:14,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/f7352ec59d1f450e9a60746d646c44a3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T09:27:14,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 41bd7632cda810f44aae6fb0260d0e2d in 1292ms, sequenceid=12, compaction requested=false 2024-11-10T09:27:14,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:14,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T09:27:14,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:14,842 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T09:27:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/19f7524415d84b80be328d6155a82071 is 50, key is test_row_0/A:col10/1731230833502/Put/seqid=0 2024-11-10T09:27:14,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742235_1411 (size=12001) 2024-11-10T09:27:15,251 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/19f7524415d84b80be328d6155a82071 2024-11-10T09:27:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/bff757222831437da60232c6318908ff is 50, key is test_row_0/B:col10/1731230833502/Put/seqid=0 2024-11-10T09:27:15,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742236_1412 (size=12001) 2024-11-10T09:27:15,260 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:27:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:27:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:15,660 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/bff757222831437da60232c6318908ff 2024-11-10T09:27:15,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/527909507d2a482894799d01ea2687b4 is 50, key is test_row_0/C:col10/1731230833502/Put/seqid=0 2024-11-10T09:27:15,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742237_1413 (size=12001) 2024-11-10T09:27:15,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230895676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230895676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,704 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:15,704 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230895703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230895703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230895703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:15,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230895816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230895825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:15,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230895825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:15,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230896026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230896035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230896035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,069 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/527909507d2a482894799d01ea2687b4 2024-11-10T09:27:16,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/19f7524415d84b80be328d6155a82071 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/19f7524415d84b80be328d6155a82071 2024-11-10T09:27:16,076 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/19f7524415d84b80be328d6155a82071, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T09:27:16,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/bff757222831437da60232c6318908ff as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/bff757222831437da60232c6318908ff 2024-11-10T09:27:16,081 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/bff757222831437da60232c6318908ff, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T09:27:16,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/527909507d2a482894799d01ea2687b4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/527909507d2a482894799d01ea2687b4 2024-11-10T09:27:16,085 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/527909507d2a482894799d01ea2687b4, entries=150, sequenceid=38, filesize=11.7 K 2024-11-10T09:27:16,085 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 41bd7632cda810f44aae6fb0260d0e2d in 1243ms, sequenceid=38, compaction requested=false 2024-11-10T09:27:16,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:16,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:16,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-10T09:27:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-10T09:27:16,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T09:27:16,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6180 sec 2024-11-10T09:27:16,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6270 sec 2024-11-10T09:27:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:16,340 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:16,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:27:16,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/31170776382040709c448f1b154350ed is 50, key is test_row_0/A:col10/1731230835663/Put/seqid=0 2024-11-10T09:27:16,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742238_1414 (size=14341) 2024-11-10T09:27:16,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230896391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230896393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230896396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230896505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230896505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230896505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230896716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230896716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230896716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:16,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:16,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/31170776382040709c448f1b154350ed 2024-11-10T09:27:16,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ea797b40658a4315b0a188a7a8104d0e is 50, key is test_row_0/B:col10/1731230835663/Put/seqid=0 2024-11-10T09:27:16,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742239_1415 (size=12001) 2024-11-10T09:27:17,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230897025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230897026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230897036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ea797b40658a4315b0a188a7a8104d0e 2024-11-10T09:27:17,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/b4f14b9616094856965e386b8e26d964 is 50, key is test_row_0/C:col10/1731230835663/Put/seqid=0 2024-11-10T09:27:17,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742240_1416 (size=12001) 2024-11-10T09:27:17,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230897546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230897546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230897546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/b4f14b9616094856965e386b8e26d964 2024-11-10T09:27:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T09:27:17,636 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:17,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-10T09:27:17,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:27:17,677 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/31170776382040709c448f1b154350ed as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/31170776382040709c448f1b154350ed 2024-11-10T09:27:17,678 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:17,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:17,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/31170776382040709c448f1b154350ed, entries=200, sequenceid=49, filesize=14.0 K 2024-11-10T09:27:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ea797b40658a4315b0a188a7a8104d0e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ea797b40658a4315b0a188a7a8104d0e 2024-11-10T09:27:17,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ea797b40658a4315b0a188a7a8104d0e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:27:17,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/b4f14b9616094856965e386b8e26d964 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/b4f14b9616094856965e386b8e26d964 2024-11-10T09:27:17,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/b4f14b9616094856965e386b8e26d964, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T09:27:17,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 41bd7632cda810f44aae6fb0260d0e2d in 1349ms, sequenceid=49, compaction requested=true 2024-11-10T09:27:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:17,689 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:17,689 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:17,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/A is initiating minor compaction (all files) 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/B is initiating minor compaction (all files) 2024-11-10T09:27:17,690 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/A in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:17,690 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/B in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:17,690 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/85d4a46f607b492692113b2d72c0ea10, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/19f7524415d84b80be328d6155a82071, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/31170776382040709c448f1b154350ed] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=37.4 K 2024-11-10T09:27:17,690 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ad37bbf4f1a242ff913bc60659a80f98, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/bff757222831437da60232c6318908ff, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ea797b40658a4315b0a188a7a8104d0e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=35.2 K 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d4a46f607b492692113b2d72c0ea10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731230833469 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ad37bbf4f1a242ff913bc60659a80f98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731230833469 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting bff757222831437da60232c6318908ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731230833502 2024-11-10T09:27:17,690 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f7524415d84b80be328d6155a82071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731230833502 2024-11-10T09:27:17,691 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31170776382040709c448f1b154350ed, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230835663 2024-11-10T09:27:17,691 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ea797b40658a4315b0a188a7a8104d0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230835663 2024-11-10T09:27:17,697 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#A#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:17,697 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/adb54addec3641d98d1d006325f9958a is 50, key is test_row_0/A:col10/1731230835663/Put/seqid=0 2024-11-10T09:27:17,698 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:17,698 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/d24c3c1892f043bcaedcca05d6b9d347 is 50, key is test_row_0/B:col10/1731230835663/Put/seqid=0 2024-11-10T09:27:17,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742241_1417 (size=12104) 2024-11-10T09:27:17,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742242_1418 (size=12104) 2024-11-10T09:27:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:17,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:17,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T09:27:17,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/2ed5b98ccada45bab7d7e0abdc98d7a0 is 50, key is test_row_0/A:col10/1731230837720/Put/seqid=0 2024-11-10T09:27:17,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742243_1419 (size=14341) 2024-11-10T09:27:17,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230897747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230897750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:27:17,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:17,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:17,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230897856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230897866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:17,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:17,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:17,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:17,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:17,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:17,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:27:18,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230898065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230898076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,112 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/adb54addec3641d98d1d006325f9958a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/adb54addec3641d98d1d006325f9958a 2024-11-10T09:27:18,115 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/A of 41bd7632cda810f44aae6fb0260d0e2d into adb54addec3641d98d1d006325f9958a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:18,115 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:18,115 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/A, priority=13, startTime=1731230837689; duration=0sec 2024-11-10T09:27:18,115 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:18,115 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:A 2024-11-10T09:27:18,115 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:18,116 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:18,117 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/C is initiating minor compaction (all files) 2024-11-10T09:27:18,117 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/C in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,117 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/f7352ec59d1f450e9a60746d646c44a3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/527909507d2a482894799d01ea2687b4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/b4f14b9616094856965e386b8e26d964] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=35.2 K 2024-11-10T09:27:18,118 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7352ec59d1f450e9a60746d646c44a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731230833469 2024-11-10T09:27:18,118 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 527909507d2a482894799d01ea2687b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731230833502 2024-11-10T09:27:18,118 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4f14b9616094856965e386b8e26d964, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230835663 2024-11-10T09:27:18,121 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/d24c3c1892f043bcaedcca05d6b9d347 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d24c3c1892f043bcaedcca05d6b9d347 2024-11-10T09:27:18,124 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/B of 41bd7632cda810f44aae6fb0260d0e2d into d24c3c1892f043bcaedcca05d6b9d347(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:18,124 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:18,124 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/B, priority=13, startTime=1731230837689; duration=0sec 2024-11-10T09:27:18,124 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:18,124 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:B 2024-11-10T09:27:18,124 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:18,125 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/7c56589245b64381ad7c121a70b242ef is 50, key is test_row_0/C:col10/1731230835663/Put/seqid=0 2024-11-10T09:27:18,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742244_1420 (size=12104) 2024-11-10T09:27:18,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/2ed5b98ccada45bab7d7e0abdc98d7a0 2024-11-10T09:27:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/e8f25fecf3e643c1b3a6849faf8796be is 50, key is test_row_0/B:col10/1731230837720/Put/seqid=0 2024-11-10T09:27:18,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742245_1421 (size=12001) 2024-11-10T09:27:18,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:27:18,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230898375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230898395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:18,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,532 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/7c56589245b64381ad7c121a70b242ef as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c56589245b64381ad7c121a70b242ef 2024-11-10T09:27:18,536 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/C of 41bd7632cda810f44aae6fb0260d0e2d into 7c56589245b64381ad7c121a70b242ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:18,536 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:18,536 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/C, priority=13, startTime=1731230837689; duration=0sec 2024-11-10T09:27:18,536 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:18,536 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:C 2024-11-10T09:27:18,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/e8f25fecf3e643c1b3a6849faf8796be 2024-11-10T09:27:18,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/49637b5b2dbc45ddb0c0f73246e5b93d is 50, key is test_row_0/C:col10/1731230837720/Put/seqid=0 2024-11-10T09:27:18,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742246_1422 (size=12001) 2024-11-10T09:27:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230898566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230898566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230898566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,571 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:18,571 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,572 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:18,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:18,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:27:18,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230898885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:18,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:18,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:18,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:18,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230898915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:18,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/49637b5b2dbc45ddb0c0f73246e5b93d 2024-11-10T09:27:18,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/2ed5b98ccada45bab7d7e0abdc98d7a0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2ed5b98ccada45bab7d7e0abdc98d7a0 2024-11-10T09:27:18,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2ed5b98ccada45bab7d7e0abdc98d7a0, entries=200, sequenceid=75, filesize=14.0 K 2024-11-10T09:27:18,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/e8f25fecf3e643c1b3a6849faf8796be as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e8f25fecf3e643c1b3a6849faf8796be 2024-11-10T09:27:18,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e8f25fecf3e643c1b3a6849faf8796be, entries=150, sequenceid=75, filesize=11.7 K 2024-11-10T09:27:18,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/49637b5b2dbc45ddb0c0f73246e5b93d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/49637b5b2dbc45ddb0c0f73246e5b93d 2024-11-10T09:27:18,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/49637b5b2dbc45ddb0c0f73246e5b93d, entries=150, sequenceid=75, filesize=11.7 K 2024-11-10T09:27:18,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 41bd7632cda810f44aae6fb0260d0e2d in 1255ms, sequenceid=75, compaction requested=false 2024-11-10T09:27:18,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:19,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T09:27:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:19,053 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T09:27:19,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/fecb6a72a4494c26838d5b7f331c8d42 is 50, key is test_row_0/A:col10/1731230837748/Put/seqid=0 2024-11-10T09:27:19,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742247_1423 (size=12001) 2024-11-10T09:27:19,461 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/fecb6a72a4494c26838d5b7f331c8d42 2024-11-10T09:27:19,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/9162e50cfa0a4190918d61c97e3932e8 is 50, key is test_row_0/B:col10/1731230837748/Put/seqid=0 2024-11-10T09:27:19,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742248_1424 (size=12001) 2024-11-10T09:27:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:27:19,870 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/9162e50cfa0a4190918d61c97e3932e8 2024-11-10T09:27:19,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/2d25ec3fbb1341bdae431fb223f15658 is 50, key is test_row_0/C:col10/1731230837748/Put/seqid=0 2024-11-10T09:27:19,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742249_1425 (size=12001) 2024-11-10T09:27:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:19,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230900004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230900005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230900116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230900116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,299 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/2d25ec3fbb1341bdae431fb223f15658 2024-11-10T09:27:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/fecb6a72a4494c26838d5b7f331c8d42 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/fecb6a72a4494c26838d5b7f331c8d42 2024-11-10T09:27:20,305 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/fecb6a72a4494c26838d5b7f331c8d42, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T09:27:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/9162e50cfa0a4190918d61c97e3932e8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/9162e50cfa0a4190918d61c97e3932e8 2024-11-10T09:27:20,309 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/9162e50cfa0a4190918d61c97e3932e8, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T09:27:20,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/2d25ec3fbb1341bdae431fb223f15658 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2d25ec3fbb1341bdae431fb223f15658 2024-11-10T09:27:20,314 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2d25ec3fbb1341bdae431fb223f15658, entries=150, sequenceid=88, filesize=11.7 K 2024-11-10T09:27:20,314 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 41bd7632cda810f44aae6fb0260d0e2d in 1262ms, sequenceid=88, compaction requested=true 2024-11-10T09:27:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-10T09:27:20,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-10T09:27:20,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T09:27:20,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-11-10T09:27:20,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.6810 sec 2024-11-10T09:27:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:20,333 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:20,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T09:27:20,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/2dd737f964894a90b3e041b09ddebdd4 is 50, key is test_row_0/A:col10/1731230839979/Put/seqid=0 2024-11-10T09:27:20,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742250_1426 (size=14341) 2024-11-10T09:27:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230900369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230900369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230900486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230900486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230900585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,593 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230900596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230900596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,602 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,602 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230900696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230900706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:20,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:20,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/2dd737f964894a90b3e041b09ddebdd4 2024-11-10T09:27:20,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/f1e9e7953b184f52895177355fbeb418 is 50, key is test_row_0/B:col10/1731230839979/Put/seqid=0 2024-11-10T09:27:20,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742251_1427 (size=12001) 2024-11-10T09:27:21,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230901005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:21,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230901015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:21,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/f1e9e7953b184f52895177355fbeb418 2024-11-10T09:27:21,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/7813542b42444d739c4d688d82f17417 is 50, key is test_row_0/C:col10/1731230839979/Put/seqid=0 2024-11-10T09:27:21,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742252_1428 (size=12001) 2024-11-10T09:27:21,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230901515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:21,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230901525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:21,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/7813542b42444d739c4d688d82f17417 2024-11-10T09:27:21,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/2dd737f964894a90b3e041b09ddebdd4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2dd737f964894a90b3e041b09ddebdd4 2024-11-10T09:27:21,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2dd737f964894a90b3e041b09ddebdd4, entries=200, sequenceid=115, filesize=14.0 K 2024-11-10T09:27:21,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/f1e9e7953b184f52895177355fbeb418 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/f1e9e7953b184f52895177355fbeb418 2024-11-10T09:27:21,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/f1e9e7953b184f52895177355fbeb418, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T09:27:21,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/7813542b42444d739c4d688d82f17417 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7813542b42444d739c4d688d82f17417 2024-11-10T09:27:21,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7813542b42444d739c4d688d82f17417, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T09:27:21,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 41bd7632cda810f44aae6fb0260d0e2d in 1246ms, sequenceid=115, compaction requested=true 2024-11-10T09:27:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:21,579 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:21,579 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:21,580 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:21,580 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:21,580 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/A is initiating minor compaction (all files) 2024-11-10T09:27:21,580 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/B is initiating minor compaction (all files) 2024-11-10T09:27:21,580 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/A in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:21,580 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/B in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:21,580 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/adb54addec3641d98d1d006325f9958a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2ed5b98ccada45bab7d7e0abdc98d7a0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/fecb6a72a4494c26838d5b7f331c8d42, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2dd737f964894a90b3e041b09ddebdd4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=51.5 K 2024-11-10T09:27:21,580 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d24c3c1892f043bcaedcca05d6b9d347, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e8f25fecf3e643c1b3a6849faf8796be, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/9162e50cfa0a4190918d61c97e3932e8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/f1e9e7953b184f52895177355fbeb418] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=47.0 K 2024-11-10T09:27:21,580 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb54addec3641d98d1d006325f9958a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230835663 2024-11-10T09:27:21,580 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d24c3c1892f043bcaedcca05d6b9d347, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230835663 2024-11-10T09:27:21,580 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ed5b98ccada45bab7d7e0abdc98d7a0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731230836386 2024-11-10T09:27:21,581 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f25fecf3e643c1b3a6849faf8796be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731230836386 2024-11-10T09:27:21,581 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting fecb6a72a4494c26838d5b7f331c8d42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731230837731 2024-11-10T09:27:21,581 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 9162e50cfa0a4190918d61c97e3932e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731230837731 2024-11-10T09:27:21,581 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dd737f964894a90b3e041b09ddebdd4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230839979 2024-11-10T09:27:21,581 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e9e7953b184f52895177355fbeb418, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230839979 2024-11-10T09:27:21,589 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#B#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:21,589 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/4f8c1799ae7d45269275ec93da13fea5 is 50, key is test_row_0/B:col10/1731230839979/Put/seqid=0 2024-11-10T09:27:21,596 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#A#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:21,596 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d8661232e27d4aaab41e52ae99a90dfd is 50, key is test_row_0/A:col10/1731230839979/Put/seqid=0 2024-11-10T09:27:21,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742253_1429 (size=12241) 2024-11-10T09:27:21,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742254_1430 (size=12241) 2024-11-10T09:27:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T09:27:21,846 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:21,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-10T09:27:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:27:21,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:21,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:21,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:27:22,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T09:27:22,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:22,000 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-10T09:27:22,001 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/4f8c1799ae7d45269275ec93da13fea5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/4f8c1799ae7d45269275ec93da13fea5 2024-11-10T09:27:22,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/75574027412e48ec8666af7fb73ca504 is 50, key is test_row_0/A:col10/1731230840367/Put/seqid=0 2024-11-10T09:27:22,010 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d8661232e27d4aaab41e52ae99a90dfd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d8661232e27d4aaab41e52ae99a90dfd 2024-11-10T09:27:22,012 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/B of 41bd7632cda810f44aae6fb0260d0e2d into 4f8c1799ae7d45269275ec93da13fea5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:22,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:22,012 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/B, priority=12, startTime=1731230841579; duration=0sec 2024-11-10T09:27:22,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:22,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:B 2024-11-10T09:27:22,012 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:22,013 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:22,013 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/C is initiating minor compaction (all files) 2024-11-10T09:27:22,013 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/C in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:22,013 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c56589245b64381ad7c121a70b242ef, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/49637b5b2dbc45ddb0c0f73246e5b93d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2d25ec3fbb1341bdae431fb223f15658, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7813542b42444d739c4d688d82f17417] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=47.0 K 2024-11-10T09:27:22,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c56589245b64381ad7c121a70b242ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731230835663 2024-11-10T09:27:22,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 49637b5b2dbc45ddb0c0f73246e5b93d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731230836386 2024-11-10T09:27:22,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d25ec3fbb1341bdae431fb223f15658, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731230837731 2024-11-10T09:27:22,015 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7813542b42444d739c4d688d82f17417, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230839979 2024-11-10T09:27:22,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742255_1431 (size=12001) 2024-11-10T09:27:22,026 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/A of 41bd7632cda810f44aae6fb0260d0e2d into d8661232e27d4aaab41e52ae99a90dfd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:22,026 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:22,026 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/A, priority=12, startTime=1731230841579; duration=0sec 2024-11-10T09:27:22,026 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:22,026 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:A 2024-11-10T09:27:22,027 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:22,028 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/d2b92c270d504ca8ab5b5168ddb9500c is 50, key is test_row_0/C:col10/1731230839979/Put/seqid=0 2024-11-10T09:27:22,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742256_1432 (size=12241) 2024-11-10T09:27:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:27:22,418 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/75574027412e48ec8666af7fb73ca504 2024-11-10T09:27:22,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/02c12c03f6e5438a95fb735af9e41cb6 is 50, key is test_row_0/B:col10/1731230840367/Put/seqid=0 2024-11-10T09:27:22,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742257_1433 (size=12001) 2024-11-10T09:27:22,435 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/d2b92c270d504ca8ab5b5168ddb9500c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/d2b92c270d504ca8ab5b5168ddb9500c 2024-11-10T09:27:22,438 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/C of 41bd7632cda810f44aae6fb0260d0e2d into d2b92c270d504ca8ab5b5168ddb9500c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:22,438 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:22,438 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/C, priority=12, startTime=1731230841579; duration=0sec 2024-11-10T09:27:22,438 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:22,438 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:C 2024-11-10T09:27:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:27:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:22,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:22,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230902608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:22,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230902614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:22,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230902725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230902726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:22,829 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/02c12c03f6e5438a95fb735af9e41cb6 2024-11-10T09:27:22,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/fe1a374642ae447791d4b5da81573d4a is 50, key is test_row_0/C:col10/1731230840367/Put/seqid=0 2024-11-10T09:27:22,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742258_1434 (size=12001) 2024-11-10T09:27:22,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230902935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230902935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:27:23,241 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/fe1a374642ae447791d4b5da81573d4a 2024-11-10T09:27:23,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/75574027412e48ec8666af7fb73ca504 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/75574027412e48ec8666af7fb73ca504 2024-11-10T09:27:23,247 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/75574027412e48ec8666af7fb73ca504, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T09:27:23,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/02c12c03f6e5438a95fb735af9e41cb6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/02c12c03f6e5438a95fb735af9e41cb6 2024-11-10T09:27:23,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230903246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,251 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/02c12c03f6e5438a95fb735af9e41cb6, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T09:27:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:23,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/fe1a374642ae447791d4b5da81573d4a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/fe1a374642ae447791d4b5da81573d4a 2024-11-10T09:27:23,254 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/fe1a374642ae447791d4b5da81573d4a, entries=150, sequenceid=124, filesize=11.7 K 2024-11-10T09:27:23,254 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 41bd7632cda810f44aae6fb0260d0e2d in 1254ms, sequenceid=124, compaction requested=false 2024-11-10T09:27:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:23,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-10T09:27:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-10T09:27:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:23,256 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:23,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-10T09:27:23,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T09:27:23,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-10T09:27:23,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4120 sec 2024-11-10T09:27:23,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/3a0dd8f46b9d40a898629cfbe5e49b09 is 50, key is test_row_0/A:col10/1731230842605/Put/seqid=0 2024-11-10T09:27:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742259_1435 (size=14541) 2024-11-10T09:27:23,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/3a0dd8f46b9d40a898629cfbe5e49b09 2024-11-10T09:27:23,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/696ca29bc7034a768b87fab150bdd19a is 50, key is test_row_0/B:col10/1731230842605/Put/seqid=0 2024-11-10T09:27:23,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742260_1436 (size=12151) 2024-11-10T09:27:23,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230903280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:23,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230903396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:23,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230903605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:23,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/696ca29bc7034a768b87fab150bdd19a 2024-11-10T09:27:23,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/157b09c6961d4f4f94d3fb840adb7632 is 50, key is test_row_0/C:col10/1731230842605/Put/seqid=0 2024-11-10T09:27:23,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742261_1437 (size=12151) 2024-11-10T09:27:23,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230903756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:23,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230903916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T09:27:23,996 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:23,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-10T09:27:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:27:23,998 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:23,998 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:23,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:24,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/157b09c6961d4f4f94d3fb840adb7632 2024-11-10T09:27:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/3a0dd8f46b9d40a898629cfbe5e49b09 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/3a0dd8f46b9d40a898629cfbe5e49b09 2024-11-10T09:27:24,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/3a0dd8f46b9d40a898629cfbe5e49b09, entries=200, sequenceid=155, filesize=14.2 K 2024-11-10T09:27:24,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/696ca29bc7034a768b87fab150bdd19a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/696ca29bc7034a768b87fab150bdd19a 2024-11-10T09:27:24,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/696ca29bc7034a768b87fab150bdd19a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T09:27:24,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/157b09c6961d4f4f94d3fb840adb7632 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/157b09c6961d4f4f94d3fb840adb7632 2024-11-10T09:27:24,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/157b09c6961d4f4f94d3fb840adb7632, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T09:27:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:27:24,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 41bd7632cda810f44aae6fb0260d0e2d in 850ms, sequenceid=155, compaction requested=true 2024-11-10T09:27:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:24,106 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:24,106 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:24,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/B is initiating minor compaction (all files) 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/A is initiating minor compaction (all files) 2024-11-10T09:27:24,107 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/B in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:24,107 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/A in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:24,107 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/4f8c1799ae7d45269275ec93da13fea5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/02c12c03f6e5438a95fb735af9e41cb6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/696ca29bc7034a768b87fab150bdd19a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=35.5 K 2024-11-10T09:27:24,107 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d8661232e27d4aaab41e52ae99a90dfd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/75574027412e48ec8666af7fb73ca504, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/3a0dd8f46b9d40a898629cfbe5e49b09] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=37.9 K 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8c1799ae7d45269275ec93da13fea5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230839979 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8661232e27d4aaab41e52ae99a90dfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230839979 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c12c03f6e5438a95fb735af9e41cb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731230840354 2024-11-10T09:27:24,107 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75574027412e48ec8666af7fb73ca504, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731230840354 2024-11-10T09:27:24,108 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 696ca29bc7034a768b87fab150bdd19a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731230842605 2024-11-10T09:27:24,108 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a0dd8f46b9d40a898629cfbe5e49b09, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731230842605 2024-11-10T09:27:24,113 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#A#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:24,113 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/13abd25ede5d4800a792e1bcd535a6ca is 50, key is test_row_0/A:col10/1731230842605/Put/seqid=0 2024-11-10T09:27:24,113 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:24,113 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/36efe123588042688378bc971f15c53e is 50, key is test_row_0/B:col10/1731230842605/Put/seqid=0 2024-11-10T09:27:24,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742262_1438 (size=12493) 2024-11-10T09:27:24,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742263_1439 (size=12493) 2024-11-10T09:27:24,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T09:27:24,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:24,150 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-10T09:27:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/efe0d198145f4316ba12f0c2c2560231 is 50, key is test_row_0/A:col10/1731230843262/Put/seqid=0 2024-11-10T09:27:24,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742264_1440 (size=12151) 2024-11-10T09:27:24,160 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/efe0d198145f4316ba12f0c2c2560231 2024-11-10T09:27:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/774dd133f7744c05a1b8304d3255b47b is 50, key is test_row_0/B:col10/1731230843262/Put/seqid=0 2024-11-10T09:27:24,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742265_1441 (size=12151) 2024-11-10T09:27:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:27:24,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:24,521 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/36efe123588042688378bc971f15c53e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/36efe123588042688378bc971f15c53e 2024-11-10T09:27:24,521 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/13abd25ede5d4800a792e1bcd535a6ca as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/13abd25ede5d4800a792e1bcd535a6ca 2024-11-10T09:27:24,525 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/B of 41bd7632cda810f44aae6fb0260d0e2d into 36efe123588042688378bc971f15c53e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:24,525 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/A of 41bd7632cda810f44aae6fb0260d0e2d into 13abd25ede5d4800a792e1bcd535a6ca(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:24,525 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:24,525 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:24,525 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/A, priority=13, startTime=1731230844106; duration=0sec 2024-11-10T09:27:24,525 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/B, priority=13, startTime=1731230844106; duration=0sec 2024-11-10T09:27:24,526 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:24,526 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:A 2024-11-10T09:27:24,526 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:24,526 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:24,526 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:B 2024-11-10T09:27:24,526 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:24,526 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/C is initiating minor compaction (all files) 2024-11-10T09:27:24,527 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/C in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:24,527 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/d2b92c270d504ca8ab5b5168ddb9500c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/fe1a374642ae447791d4b5da81573d4a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/157b09c6961d4f4f94d3fb840adb7632] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=35.5 K 2024-11-10T09:27:24,527 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b92c270d504ca8ab5b5168ddb9500c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230839979 2024-11-10T09:27:24,527 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1a374642ae447791d4b5da81573d4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731230840354 2024-11-10T09:27:24,528 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 157b09c6961d4f4f94d3fb840adb7632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731230842605 2024-11-10T09:27:24,532 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:24,533 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/5a881583bf5246fab588a3b0bcabbbb1 is 50, key is test_row_0/C:col10/1731230842605/Put/seqid=0 2024-11-10T09:27:24,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742266_1442 (size=12493) 2024-11-10T09:27:24,542 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/5a881583bf5246fab588a3b0bcabbbb1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5a881583bf5246fab588a3b0bcabbbb1 2024-11-10T09:27:24,545 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/C of 41bd7632cda810f44aae6fb0260d0e2d into 5a881583bf5246fab588a3b0bcabbbb1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:24,546 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:24,546 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/C, priority=13, startTime=1731230844106; duration=0sec 2024-11-10T09:27:24,546 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:24,546 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:C 2024-11-10T09:27:24,568 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/774dd133f7744c05a1b8304d3255b47b 2024-11-10T09:27:24,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/90757160c4914c5eb0e4745aea41e3c0 is 50, key is test_row_0/C:col10/1731230843262/Put/seqid=0 2024-11-10T09:27:24,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742267_1443 (size=12151) 2024-11-10T09:27:24,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230904588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:24,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48844 deadline: 1731230904606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:24,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48804 deadline: 1731230904606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:24,611 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:24,611 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:24,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48820 deadline: 1731230904616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:24,621 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_0' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:27:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230904706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:24,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230904775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:24,782 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230904916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:24,978 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/90757160c4914c5eb0e4745aea41e3c0 2024-11-10T09:27:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/efe0d198145f4316ba12f0c2c2560231 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/efe0d198145f4316ba12f0c2c2560231 2024-11-10T09:27:24,985 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/efe0d198145f4316ba12f0c2c2560231, entries=150, sequenceid=163, filesize=11.9 K 2024-11-10T09:27:24,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/774dd133f7744c05a1b8304d3255b47b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/774dd133f7744c05a1b8304d3255b47b 2024-11-10T09:27:24,988 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/774dd133f7744c05a1b8304d3255b47b, entries=150, sequenceid=163, filesize=11.9 K 2024-11-10T09:27:24,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/90757160c4914c5eb0e4745aea41e3c0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/90757160c4914c5eb0e4745aea41e3c0 2024-11-10T09:27:24,991 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/90757160c4914c5eb0e4745aea41e3c0, entries=150, sequenceid=163, filesize=11.9 K 2024-11-10T09:27:24,992 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 41bd7632cda810f44aae6fb0260d0e2d in 842ms, sequenceid=163, compaction requested=false 2024-11-10T09:27:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-10T09:27:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-10T09:27:24,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T09:27:24,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-10T09:27:24,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 999 msec 2024-11-10T09:27:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T09:27:25,136 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:25,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-10T09:27:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:27:25,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:25,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:25,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:25,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:25,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:25,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-10T09:27:25,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/e7faa3a7a0a2424294a0503b3c7ff60d is 50, key is test_row_0/A:col10/1731230845230/Put/seqid=0 2024-11-10T09:27:25,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742268_1444 (size=14541) 2024-11-10T09:27:25,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/e7faa3a7a0a2424294a0503b3c7ff60d 2024-11-10T09:27:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:27:25,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/acf14d67d0504261974276e42cd1ae60 is 50, key is test_row_0/B:col10/1731230845230/Put/seqid=0 2024-11-10T09:27:25,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742269_1445 (size=12151) 2024-11-10T09:27:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230905259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:25,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:27:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:25,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230905365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:25,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:27:25,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:25,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:27:25,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230905575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:25,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:27:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:25,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/acf14d67d0504261974276e42cd1ae60 2024-11-10T09:27:25,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/aae9a47315764fbebc4a2d8d4a5be08b is 50, key is test_row_0/C:col10/1731230845230/Put/seqid=0 2024-11-10T09:27:25,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742270_1446 (size=12151) 2024-11-10T09:27:25,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:27:25,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:25,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:27:25,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230905886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:25,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:27:25,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:25,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:25,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:26,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:27:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:26,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:26,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/aae9a47315764fbebc4a2d8d4a5be08b 2024-11-10T09:27:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/e7faa3a7a0a2424294a0503b3c7ff60d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/e7faa3a7a0a2424294a0503b3c7ff60d 2024-11-10T09:27:26,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/e7faa3a7a0a2424294a0503b3c7ff60d, entries=200, sequenceid=195, filesize=14.2 K 2024-11-10T09:27:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/acf14d67d0504261974276e42cd1ae60 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/acf14d67d0504261974276e42cd1ae60 2024-11-10T09:27:26,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/acf14d67d0504261974276e42cd1ae60, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:27:26,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/aae9a47315764fbebc4a2d8d4a5be08b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aae9a47315764fbebc4a2d8d4a5be08b 2024-11-10T09:27:26,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aae9a47315764fbebc4a2d8d4a5be08b, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:27:26,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 41bd7632cda810f44aae6fb0260d0e2d in 845ms, sequenceid=195, compaction requested=true 2024-11-10T09:27:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:26,077 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:26,077 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:26,078 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:26,078 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/A is initiating minor compaction (all files) 2024-11-10T09:27:26,078 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/A in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:26,078 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/13abd25ede5d4800a792e1bcd535a6ca, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/efe0d198145f4316ba12f0c2c2560231, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/e7faa3a7a0a2424294a0503b3c7ff60d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=38.3 K 2024-11-10T09:27:26,078 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:26,078 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/B is initiating minor compaction (all files) 2024-11-10T09:27:26,078 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/B in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:26,078 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/36efe123588042688378bc971f15c53e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/774dd133f7744c05a1b8304d3255b47b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/acf14d67d0504261974276e42cd1ae60] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=35.9 K 2024-11-10T09:27:26,078 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13abd25ede5d4800a792e1bcd535a6ca, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731230842605 2024-11-10T09:27:26,079 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 36efe123588042688378bc971f15c53e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731230842605 2024-11-10T09:27:26,079 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting efe0d198145f4316ba12f0c2c2560231, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731230843262 2024-11-10T09:27:26,079 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 774dd133f7744c05a1b8304d3255b47b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731230843262 2024-11-10T09:27:26,079 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7faa3a7a0a2424294a0503b3c7ff60d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230844576 2024-11-10T09:27:26,079 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting acf14d67d0504261974276e42cd1ae60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230844576 2024-11-10T09:27:26,084 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:26,085 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/c0955c40e91e42d79b8ca9cc10994858 is 50, key is test_row_0/B:col10/1731230845230/Put/seqid=0 2024-11-10T09:27:26,087 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#A#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:26,087 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/af23c7a257494fc4baeedcd3c018bb5a is 50, key is test_row_0/A:col10/1731230845230/Put/seqid=0 2024-11-10T09:27:26,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742271_1447 (size=12595) 2024-11-10T09:27:26,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742272_1448 (size=12595) 2024-11-10T09:27:26,099 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/af23c7a257494fc4baeedcd3c018bb5a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/af23c7a257494fc4baeedcd3c018bb5a 2024-11-10T09:27:26,102 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/A of 41bd7632cda810f44aae6fb0260d0e2d into af23c7a257494fc4baeedcd3c018bb5a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:26,102 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:26,102 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/A, priority=13, startTime=1731230846077; duration=0sec 2024-11-10T09:27:26,102 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:26,102 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:A 2024-11-10T09:27:26,102 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:26,103 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:26,103 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/C is initiating minor compaction (all files) 2024-11-10T09:27:26,103 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/C in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:26,103 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5a881583bf5246fab588a3b0bcabbbb1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/90757160c4914c5eb0e4745aea41e3c0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aae9a47315764fbebc4a2d8d4a5be08b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=35.9 K 2024-11-10T09:27:26,104 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a881583bf5246fab588a3b0bcabbbb1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731230842605 2024-11-10T09:27:26,104 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90757160c4914c5eb0e4745aea41e3c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731230843262 2024-11-10T09:27:26,104 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting aae9a47315764fbebc4a2d8d4a5be08b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230844576 2024-11-10T09:27:26,110 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#C#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:26,110 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/e237cd15cde241378fce05a1a0155dfd is 50, key is test_row_0/C:col10/1731230845230/Put/seqid=0 2024-11-10T09:27:26,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742273_1449 (size=12595) 2024-11-10T09:27:26,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-10T09:27:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:26,206 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-10T09:27:26,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/86a8e9bb09c24455b0a4a1bf4b349b83 is 50, key is test_row_1/A:col10/1731230845240/Put/seqid=0 2024-11-10T09:27:26,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742274_1450 (size=9757) 2024-11-10T09:27:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:27:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:26,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:26,493 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/c0955c40e91e42d79b8ca9cc10994858 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/c0955c40e91e42d79b8ca9cc10994858 2024-11-10T09:27:26,496 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/B of 41bd7632cda810f44aae6fb0260d0e2d into c0955c40e91e42d79b8ca9cc10994858(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:26,496 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:26,496 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/B, priority=13, startTime=1731230846077; duration=0sec 2024-11-10T09:27:26,496 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:26,496 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:B 2024-11-10T09:27:26,517 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/e237cd15cde241378fce05a1a0155dfd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/e237cd15cde241378fce05a1a0155dfd 2024-11-10T09:27:26,521 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/C of 41bd7632cda810f44aae6fb0260d0e2d into e237cd15cde241378fce05a1a0155dfd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:26,521 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:26,521 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/C, priority=13, startTime=1731230846077; duration=0sec 2024-11-10T09:27:26,521 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:26,521 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:C 2024-11-10T09:27:26,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230906547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:26,618 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/86a8e9bb09c24455b0a4a1bf4b349b83 2024-11-10T09:27:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/a22e56e4a4c54beb87005d3d5a3068d3 is 50, key is test_row_1/B:col10/1731230845240/Put/seqid=0 2024-11-10T09:27:26,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742275_1451 (size=9757) 2024-11-10T09:27:26,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230906655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:26,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230906796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:26,801 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:26,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230906865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:27,027 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/a22e56e4a4c54beb87005d3d5a3068d3 2024-11-10T09:27:27,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/66dacd561a1845618b24ed6f2adcdf19 is 50, key is test_row_1/C:col10/1731230845240/Put/seqid=0 2024-11-10T09:27:27,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742276_1452 (size=9757) 2024-11-10T09:27:27,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230907176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:27:27,437 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/66dacd561a1845618b24ed6f2adcdf19 2024-11-10T09:27:27,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/86a8e9bb09c24455b0a4a1bf4b349b83 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/86a8e9bb09c24455b0a4a1bf4b349b83 2024-11-10T09:27:27,444 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/86a8e9bb09c24455b0a4a1bf4b349b83, entries=100, sequenceid=203, filesize=9.5 K 2024-11-10T09:27:27,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/a22e56e4a4c54beb87005d3d5a3068d3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/a22e56e4a4c54beb87005d3d5a3068d3 2024-11-10T09:27:27,448 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/a22e56e4a4c54beb87005d3d5a3068d3, entries=100, sequenceid=203, filesize=9.5 K 2024-11-10T09:27:27,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/66dacd561a1845618b24ed6f2adcdf19 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/66dacd561a1845618b24ed6f2adcdf19 2024-11-10T09:27:27,452 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/66dacd561a1845618b24ed6f2adcdf19, entries=100, sequenceid=203, filesize=9.5 K 2024-11-10T09:27:27,452 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 41bd7632cda810f44aae6fb0260d0e2d in 1246ms, sequenceid=203, compaction requested=false 2024-11-10T09:27:27,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:27,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:27,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-10T09:27:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-10T09:27:27,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T09:27:27,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3160 sec 2024-11-10T09:27:27,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.3190 sec 2024-11-10T09:27:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:27,749 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:27,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-10T09:27:27,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/6a5e2f8d51f44a34aac60fb53c9bf3e2 is 50, key is test_row_0/A:col10/1731230846545/Put/seqid=0 2024-11-10T09:27:27,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742277_1453 (size=14541) 2024-11-10T09:27:27,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230907763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:27,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230907875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:28,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230908086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:28,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:28,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/6a5e2f8d51f44a34aac60fb53c9bf3e2 2024-11-10T09:27:28,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/eee0b71a8d4b448f9d9a3b717a11a521 is 50, key is test_row_0/B:col10/1731230846545/Put/seqid=0 2024-11-10T09:27:28,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742278_1454 (size=12151) 2024-11-10T09:27:28,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230908395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:28,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/eee0b71a8d4b448f9d9a3b717a11a521 2024-11-10T09:27:28,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/5c4ce9d5675e4c16954d86d228964aa7 is 50, key is test_row_0/C:col10/1731230846545/Put/seqid=0 2024-11-10T09:27:28,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742279_1455 (size=12151) 2024-11-10T09:27:28,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230908916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:28,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/5c4ce9d5675e4c16954d86d228964aa7 2024-11-10T09:27:28,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/6a5e2f8d51f44a34aac60fb53c9bf3e2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/6a5e2f8d51f44a34aac60fb53c9bf3e2 2024-11-10T09:27:28,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/6a5e2f8d51f44a34aac60fb53c9bf3e2, entries=200, sequenceid=235, filesize=14.2 K 2024-11-10T09:27:28,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/eee0b71a8d4b448f9d9a3b717a11a521 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/eee0b71a8d4b448f9d9a3b717a11a521 2024-11-10T09:27:28,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/eee0b71a8d4b448f9d9a3b717a11a521, entries=150, sequenceid=235, filesize=11.9 K 2024-11-10T09:27:28,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/5c4ce9d5675e4c16954d86d228964aa7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5c4ce9d5675e4c16954d86d228964aa7 2024-11-10T09:27:28,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5c4ce9d5675e4c16954d86d228964aa7, entries=150, sequenceid=235, filesize=11.9 K 2024-11-10T09:27:28,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 41bd7632cda810f44aae6fb0260d0e2d in 1249ms, sequenceid=235, compaction requested=true 2024-11-10T09:27:28,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:28,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:28,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:28,998 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:28,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:28,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:28,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:28,998 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:28,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:28,999 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:28,999 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36893 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:28,999 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/B is initiating minor compaction (all files) 2024-11-10T09:27:28,999 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/A is initiating minor compaction (all files) 2024-11-10T09:27:28,999 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/B in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:28,999 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/A in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:28,999 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/c0955c40e91e42d79b8ca9cc10994858, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/a22e56e4a4c54beb87005d3d5a3068d3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/eee0b71a8d4b448f9d9a3b717a11a521] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=33.7 K 2024-11-10T09:27:28,999 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/af23c7a257494fc4baeedcd3c018bb5a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/86a8e9bb09c24455b0a4a1bf4b349b83, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/6a5e2f8d51f44a34aac60fb53c9bf3e2] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=36.0 K 2024-11-10T09:27:28,999 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting c0955c40e91e42d79b8ca9cc10994858, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230844576 2024-11-10T09:27:28,999 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting af23c7a257494fc4baeedcd3c018bb5a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230844576 2024-11-10T09:27:29,000 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a22e56e4a4c54beb87005d3d5a3068d3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230845240 2024-11-10T09:27:29,000 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a8e9bb09c24455b0a4a1bf4b349b83, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230845240 2024-11-10T09:27:29,000 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting eee0b71a8d4b448f9d9a3b717a11a521, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230846539 2024-11-10T09:27:29,000 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5e2f8d51f44a34aac60fb53c9bf3e2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230846524 2024-11-10T09:27:29,006 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:29,006 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#A#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:29,006 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/0f71ec76644949b38413c9d30e7d268c is 50, key is test_row_0/A:col10/1731230846545/Put/seqid=0 2024-11-10T09:27:29,006 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/d07bfaa5e2e24a5fbe5b01b9eec65471 is 50, key is test_row_0/B:col10/1731230846545/Put/seqid=0 2024-11-10T09:27:29,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742281_1457 (size=12697) 2024-11-10T09:27:29,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742280_1456 (size=12697) 2024-11-10T09:27:29,021 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/0f71ec76644949b38413c9d30e7d268c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/0f71ec76644949b38413c9d30e7d268c 2024-11-10T09:27:29,025 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/A of 41bd7632cda810f44aae6fb0260d0e2d into 0f71ec76644949b38413c9d30e7d268c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:29,025 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:29,025 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/A, priority=13, startTime=1731230848998; duration=0sec 2024-11-10T09:27:29,025 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:29,025 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:A 2024-11-10T09:27:29,025 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:29,026 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:29,026 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/C is initiating minor compaction (all files) 2024-11-10T09:27:29,026 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/C in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:29,026 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/e237cd15cde241378fce05a1a0155dfd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/66dacd561a1845618b24ed6f2adcdf19, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5c4ce9d5675e4c16954d86d228964aa7] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=33.7 K 2024-11-10T09:27:29,028 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e237cd15cde241378fce05a1a0155dfd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230844576 2024-11-10T09:27:29,028 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66dacd561a1845618b24ed6f2adcdf19, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731230845240 2024-11-10T09:27:29,029 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4ce9d5675e4c16954d86d228964aa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230846539 2024-11-10T09:27:29,042 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#C#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:29,043 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/2b5189c6ca8445d3987110ec9abe7ffd is 50, key is test_row_0/C:col10/1731230846545/Put/seqid=0 2024-11-10T09:27:29,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742282_1458 (size=12697) 2024-11-10T09:27:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T09:27:29,306 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:29,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-10T09:27:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:27:29,308 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:29,308 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:29,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:27:29,423 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/d07bfaa5e2e24a5fbe5b01b9eec65471 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d07bfaa5e2e24a5fbe5b01b9eec65471 2024-11-10T09:27:29,426 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/B of 41bd7632cda810f44aae6fb0260d0e2d into d07bfaa5e2e24a5fbe5b01b9eec65471(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:29,426 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:29,426 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/B, priority=13, startTime=1731230848998; duration=0sec 2024-11-10T09:27:29,427 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:29,427 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:B 2024-11-10T09:27:29,458 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/2b5189c6ca8445d3987110ec9abe7ffd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2b5189c6ca8445d3987110ec9abe7ffd 2024-11-10T09:27:29,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-10T09:27:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:29,460 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-10T09:27:29,462 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/C of 41bd7632cda810f44aae6fb0260d0e2d into 2b5189c6ca8445d3987110ec9abe7ffd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:29,462 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:29,462 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/C, priority=13, startTime=1731230848998; duration=0sec 2024-11-10T09:27:29,463 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:29,463 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:C 2024-11-10T09:27:29,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/9bd99b5460944d81968afa753f0de91b is 50, key is test_row_0/A:col10/1731230847759/Put/seqid=0 2024-11-10T09:27:29,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742283_1459 (size=9757) 2024-11-10T09:27:29,476 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/9bd99b5460944d81968afa753f0de91b 2024-11-10T09:27:29,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ee5a16e97f0940ac9dd43e73c5bd08d8 is 50, key is test_row_0/B:col10/1731230847759/Put/seqid=0 2024-11-10T09:27:29,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742284_1460 (size=9757) 2024-11-10T09:27:29,492 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ee5a16e97f0940ac9dd43e73c5bd08d8 2024-11-10T09:27:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/aa2e5823ab9e4ca6a04c3f54a985af7a is 50, key is test_row_0/C:col10/1731230847759/Put/seqid=0 2024-11-10T09:27:29,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742285_1461 (size=9757) 2024-11-10T09:27:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:27:29,917 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/aa2e5823ab9e4ca6a04c3f54a985af7a 2024-11-10T09:27:29,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/9bd99b5460944d81968afa753f0de91b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9bd99b5460944d81968afa753f0de91b 2024-11-10T09:27:29,923 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9bd99b5460944d81968afa753f0de91b, entries=100, sequenceid=243, filesize=9.5 K 2024-11-10T09:27:29,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/ee5a16e97f0940ac9dd43e73c5bd08d8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ee5a16e97f0940ac9dd43e73c5bd08d8 2024-11-10T09:27:29,927 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ee5a16e97f0940ac9dd43e73c5bd08d8, entries=100, sequenceid=243, filesize=9.5 K 2024-11-10T09:27:29,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/aa2e5823ab9e4ca6a04c3f54a985af7a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aa2e5823ab9e4ca6a04c3f54a985af7a 2024-11-10T09:27:29,930 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aa2e5823ab9e4ca6a04c3f54a985af7a, entries=100, sequenceid=243, filesize=9.5 K 2024-11-10T09:27:29,931 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 41bd7632cda810f44aae6fb0260d0e2d in 471ms, sequenceid=243, compaction requested=false 2024-11-10T09:27:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-10T09:27:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-10T09:27:29,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T09:27:29,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 625 msec 2024-11-10T09:27:29,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 629 msec 2024-11-10T09:27:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-10T09:27:29,936 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:29,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-10T09:27:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:27:29,938 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:29,939 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:29,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:29,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:29,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:27:29,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d18e518def4e45b18369784244063600 is 50, key is test_row_0/A:col10/1731230849952/Put/seqid=0 2024-11-10T09:27:29,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742286_1462 (size=16931) 2024-11-10T09:27:30,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:27:30,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230910085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:30,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230910196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:30,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:30,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:30,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:27:30,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d18e518def4e45b18369784244063600 2024-11-10T09:27:30,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/e5ec32470c7646ff890eb4f9f1d1b0ed is 50, key is test_row_0/B:col10/1731230849952/Put/seqid=0 2024-11-10T09:27:30,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742287_1463 (size=12151) 2024-11-10T09:27:30,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230910406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:30,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:30,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:30,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:27:30,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230910715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:30,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/e5ec32470c7646ff890eb4f9f1d1b0ed 2024-11-10T09:27:30,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ee5f442709754457a6d4545939578e4d is 50, key is test_row_0/C:col10/1731230849952/Put/seqid=0 2024-11-10T09:27:30,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742288_1464 (size=12151) 2024-11-10T09:27:30,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48774 deadline: 1731230910826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:30,829 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in 41bd7632cda810f44aae6fb0260d0e2d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:30,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:30,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:31,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:31,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:27:31,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ee5f442709754457a6d4545939578e4d 2024-11-10T09:27:31,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d18e518def4e45b18369784244063600 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d18e518def4e45b18369784244063600 2024-11-10T09:27:31,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d18e518def4e45b18369784244063600, entries=250, sequenceid=255, filesize=16.5 K 2024-11-10T09:27:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/e5ec32470c7646ff890eb4f9f1d1b0ed as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e5ec32470c7646ff890eb4f9f1d1b0ed 2024-11-10T09:27:31,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e5ec32470c7646ff890eb4f9f1d1b0ed, entries=150, sequenceid=255, filesize=11.9 K 2024-11-10T09:27:31,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ee5f442709754457a6d4545939578e4d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ee5f442709754457a6d4545939578e4d 2024-11-10T09:27:31,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ee5f442709754457a6d4545939578e4d, entries=150, sequenceid=255, filesize=11.9 K 2024-11-10T09:27:31,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 41bd7632cda810f44aae6fb0260d0e2d in 1237ms, sequenceid=255, compaction requested=true 2024-11-10T09:27:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:31,193 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:31,193 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:31,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39385 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/A is initiating minor compaction (all files) 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/B is initiating minor compaction (all files) 2024-11-10T09:27:31,194 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/A in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,194 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/B in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,194 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d07bfaa5e2e24a5fbe5b01b9eec65471, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ee5a16e97f0940ac9dd43e73c5bd08d8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e5ec32470c7646ff890eb4f9f1d1b0ed] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=33.8 K 2024-11-10T09:27:31,194 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/0f71ec76644949b38413c9d30e7d268c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9bd99b5460944d81968afa753f0de91b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d18e518def4e45b18369784244063600] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=38.5 K 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d07bfaa5e2e24a5fbe5b01b9eec65471, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230846539 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f71ec76644949b38413c9d30e7d268c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230846539 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd99b5460944d81968afa753f0de91b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731230847759 2024-11-10T09:27:31,194 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5a16e97f0940ac9dd43e73c5bd08d8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731230847759 2024-11-10T09:27:31,195 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18e518def4e45b18369784244063600, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230849938 2024-11-10T09:27:31,195 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e5ec32470c7646ff890eb4f9f1d1b0ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230849938 2024-11-10T09:27:31,199 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:31,199 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/19bbb63027424693907bdc8ca2f6f6e8 is 50, key is test_row_0/B:col10/1731230849952/Put/seqid=0 2024-11-10T09:27:31,201 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:31,201 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/f5e3adc7680f4e4c8a41419643f5699f is 50, key is test_row_0/A:col10/1731230849952/Put/seqid=0 2024-11-10T09:27:31,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742289_1465 (size=12799) 2024-11-10T09:27:31,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742290_1466 (size=12799) 2024-11-10T09:27:31,208 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/19bbb63027424693907bdc8ca2f6f6e8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/19bbb63027424693907bdc8ca2f6f6e8 2024-11-10T09:27:31,208 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/f5e3adc7680f4e4c8a41419643f5699f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/f5e3adc7680f4e4c8a41419643f5699f 2024-11-10T09:27:31,211 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/B of 41bd7632cda810f44aae6fb0260d0e2d into 19bbb63027424693907bdc8ca2f6f6e8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:31,211 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/A of 41bd7632cda810f44aae6fb0260d0e2d into f5e3adc7680f4e4c8a41419643f5699f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:31,211 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:31,211 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:31,211 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/B, priority=13, startTime=1731230851193; duration=0sec 2024-11-10T09:27:31,211 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/A, priority=13, startTime=1731230851193; duration=0sec 2024-11-10T09:27:31,211 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:31,211 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:31,211 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:B 2024-11-10T09:27:31,211 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:A 2024-11-10T09:27:31,211 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:31,212 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34605 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:31,212 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/C is initiating minor compaction (all files) 2024-11-10T09:27:31,212 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/C in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,212 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2b5189c6ca8445d3987110ec9abe7ffd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aa2e5823ab9e4ca6a04c3f54a985af7a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ee5f442709754457a6d4545939578e4d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=33.8 K 2024-11-10T09:27:31,212 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b5189c6ca8445d3987110ec9abe7ffd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731230846539 2024-11-10T09:27:31,213 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting aa2e5823ab9e4ca6a04c3f54a985af7a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731230847759 2024-11-10T09:27:31,213 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5f442709754457a6d4545939578e4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230849938 2024-11-10T09:27:31,224 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:31,225 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/7c3a3c319fd949bea8ece0987241e81a is 50, key is test_row_0/C:col10/1731230849952/Put/seqid=0 2024-11-10T09:27:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:31,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742291_1467 (size=12799) 2024-11-10T09:27:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:31,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-10T09:27:31,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/9eb4d55fdc4c4d269160a0cc656d2a02 is 50, key is test_row_0/A:col10/1731230850073/Put/seqid=0 2024-11-10T09:27:31,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742292_1468 (size=14741) 2024-11-10T09:27:31,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/9eb4d55fdc4c4d269160a0cc656d2a02 2024-11-10T09:27:31,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/08b86767127a4484a858c2aedfaf9173 is 50, key is test_row_0/B:col10/1731230850073/Put/seqid=0 2024-11-10T09:27:31,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742293_1469 (size=12301) 2024-11-10T09:27:31,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230911269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:31,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:31,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:31,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230911385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:31,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230911595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:31,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:31,634 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/7c3a3c319fd949bea8ece0987241e81a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c3a3c319fd949bea8ece0987241e81a 2024-11-10T09:27:31,638 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/C of 41bd7632cda810f44aae6fb0260d0e2d into 7c3a3c319fd949bea8ece0987241e81a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:31,638 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:31,638 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/C, priority=13, startTime=1731230851193; duration=0sec 2024-11-10T09:27:31,638 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:31,638 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:C 2024-11-10T09:27:31,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/08b86767127a4484a858c2aedfaf9173 2024-11-10T09:27:31,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/20df6056dbf94a8fa873c63070119a37 is 50, key is test_row_0/C:col10/1731230850073/Put/seqid=0 2024-11-10T09:27:31,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742294_1470 (size=12301) 2024-11-10T09:27:31,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/20df6056dbf94a8fa873c63070119a37 2024-11-10T09:27:31,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/9eb4d55fdc4c4d269160a0cc656d2a02 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9eb4d55fdc4c4d269160a0cc656d2a02 2024-11-10T09:27:31,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9eb4d55fdc4c4d269160a0cc656d2a02, entries=200, sequenceid=283, filesize=14.4 K 2024-11-10T09:27:31,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/08b86767127a4484a858c2aedfaf9173 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/08b86767127a4484a858c2aedfaf9173 2024-11-10T09:27:31,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/08b86767127a4484a858c2aedfaf9173, entries=150, sequenceid=283, filesize=12.0 K 2024-11-10T09:27:31,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/20df6056dbf94a8fa873c63070119a37 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/20df6056dbf94a8fa873c63070119a37 2024-11-10T09:27:31,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/20df6056dbf94a8fa873c63070119a37, entries=150, sequenceid=283, filesize=12.0 K 2024-11-10T09:27:31,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 41bd7632cda810f44aae6fb0260d0e2d in 446ms, sequenceid=283, compaction requested=false 2024-11-10T09:27:31,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:31,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-10T09:27:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:31,770 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-10T09:27:31,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d43deabad62449a09413f6211d84324f is 50, key is test_row_0/A:col10/1731230851262/Put/seqid=0 2024-11-10T09:27:31,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742295_1471 (size=12301) 2024-11-10T09:27:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:31,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. as already flushing 2024-11-10T09:27:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230912041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:27:32,146 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:27:32,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230912155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:32,178 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d43deabad62449a09413f6211d84324f 2024-11-10T09:27:32,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/161476c5d69c4e2aa2558047528021f1 is 50, key is test_row_0/B:col10/1731230851262/Put/seqid=0 2024-11-10T09:27:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742296_1472 (size=12301) 2024-11-10T09:27:32,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230912366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:32,586 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/161476c5d69c4e2aa2558047528021f1 2024-11-10T09:27:32,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ca6e5fb2e89344069839a6a2251659fb is 50, key is test_row_0/C:col10/1731230851262/Put/seqid=0 2024-11-10T09:27:32,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742297_1473 (size=12301) 2024-11-10T09:27:32,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230912676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:32,994 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ca6e5fb2e89344069839a6a2251659fb 2024-11-10T09:27:32,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/d43deabad62449a09413f6211d84324f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d43deabad62449a09413f6211d84324f 2024-11-10T09:27:33,000 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d43deabad62449a09413f6211d84324f, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T09:27:33,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/161476c5d69c4e2aa2558047528021f1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/161476c5d69c4e2aa2558047528021f1 2024-11-10T09:27:33,003 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/161476c5d69c4e2aa2558047528021f1, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T09:27:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ca6e5fb2e89344069839a6a2251659fb as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ca6e5fb2e89344069839a6a2251659fb 2024-11-10T09:27:33,006 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ca6e5fb2e89344069839a6a2251659fb, entries=150, sequenceid=294, filesize=12.0 K 2024-11-10T09:27:33,007 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 41bd7632cda810f44aae6fb0260d0e2d in 1237ms, sequenceid=294, compaction requested=true 2024-11-10T09:27:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:33,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-10T09:27:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-10T09:27:33,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-10T09:27:33,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0690 sec 2024-11-10T09:27:33,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 3.0730 sec 2024-11-10T09:27:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:33,191 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:33,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-10T09:27:33,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8 is 50, key is test_row_0/A:col10/1731230852034/Put/seqid=0 2024-11-10T09:27:33,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742298_1474 (size=14741) 2024-11-10T09:27:33,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230913222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230913336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:33,464 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-11-10T09:27:33,465 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:33,465 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:33,465 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,466 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-10T09:27:33,466 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:33,466 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,466 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:33,468 INFO [Thread-1867 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1867. 2024-11-10T09:27:33,468 DEBUG [Thread-1867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:33,468 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,468 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:33,473 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-10T09:27:33,473 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:33,473 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,473 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:33,474 INFO [Thread-1869 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1869. 2024-11-10T09:27:33,474 DEBUG [Thread-1869 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:33,474 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,474 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:33,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:33,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230913546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:33,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8 2024-11-10T09:27:33,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/2d121c76c0ac491280fd165241b7c590 is 50, key is test_row_0/B:col10/1731230852034/Put/seqid=0 2024-11-10T09:27:33,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742299_1475 (size=12301) 2024-11-10T09:27:33,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230913856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:33,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:33,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:34,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/2d121c76c0ac491280fd165241b7c590 2024-11-10T09:27:34,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ff0f4926304f46d7be38ae9657b5baf8 is 50, key is test_row_0/C:col10/1731230852034/Put/seqid=0 2024-11-10T09:27:34,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742300_1476 (size=12301) 2024-11-10T09:27:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T09:27:34,107 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:34,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48806 deadline: 1731230914366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 , the old value is region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., hostname=802346a334f5,32883,1731230704436, seqNum=2 because the exception is null or not the one we care about 2024-11-10T09:27:34,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ff0f4926304f46d7be38ae9657b5baf8 2024-11-10T09:27:34,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8 2024-11-10T09:27:34,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8, entries=200, sequenceid=321, filesize=14.4 K 2024-11-10T09:27:34,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/2d121c76c0ac491280fd165241b7c590 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2d121c76c0ac491280fd165241b7c590 2024-11-10T09:27:34,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2d121c76c0ac491280fd165241b7c590, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T09:27:34,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/ff0f4926304f46d7be38ae9657b5baf8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ff0f4926304f46d7be38ae9657b5baf8 2024-11-10T09:27:34,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ff0f4926304f46d7be38ae9657b5baf8, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T09:27:34,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 41bd7632cda810f44aae6fb0260d0e2d in 1254ms, sequenceid=321, compaction requested=true 2024-11-10T09:27:34,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:34,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:34,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:34,445 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:34,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:34,445 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:34,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:34,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41bd7632cda810f44aae6fb0260d0e2d:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:34,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54582 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/A is initiating minor compaction (all files) 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/B is initiating minor compaction (all files) 2024-11-10T09:27:34,446 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/A in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:34,446 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/B in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:34,446 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/19bbb63027424693907bdc8ca2f6f6e8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/08b86767127a4484a858c2aedfaf9173, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/161476c5d69c4e2aa2558047528021f1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2d121c76c0ac491280fd165241b7c590] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=48.5 K 2024-11-10T09:27:34,446 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/f5e3adc7680f4e4c8a41419643f5699f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9eb4d55fdc4c4d269160a0cc656d2a02, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d43deabad62449a09413f6211d84324f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=53.3 K 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e3adc7680f4e4c8a41419643f5699f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230849938 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 19bbb63027424693907bdc8ca2f6f6e8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230849938 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eb4d55fdc4c4d269160a0cc656d2a02, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731230850073 2024-11-10T09:27:34,446 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b86767127a4484a858c2aedfaf9173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731230850073 2024-11-10T09:27:34,447 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting d43deabad62449a09413f6211d84324f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731230851249 2024-11-10T09:27:34,447 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 161476c5d69c4e2aa2558047528021f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731230851249 2024-11-10T09:27:34,447 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting a60dcfb6b0fd4f68acf20cb2ecc0c9c8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230852028 2024-11-10T09:27:34,447 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d121c76c0ac491280fd165241b7c590, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230852028 2024-11-10T09:27:34,452 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:34,452 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:34,453 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/7eb725e99316437a9d15a3ba173baa6d is 50, key is test_row_0/B:col10/1731230852034/Put/seqid=0 2024-11-10T09:27:34,453 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/c11c51d93fdc46509eddb6b6b66c675e is 50, key is test_row_0/A:col10/1731230852034/Put/seqid=0 2024-11-10T09:27:34,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742302_1478 (size=13085) 2024-11-10T09:27:34,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742301_1477 (size=13085) 2024-11-10T09:27:34,639 INFO [Thread-1854 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1854. 2024-11-10T09:27:34,639 DEBUG [Thread-1854 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:34,639 DEBUG [Thread-1854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:34,639 DEBUG [Thread-1854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:34,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:34,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:27:34,681 INFO [Thread-1856 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1856. 2024-11-10T09:27:34,681 DEBUG [Thread-1856 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:34,681 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:34,681 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:34,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:34,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/53e3bfdae3e64355a5ebbddb11c229ef is 50, key is test_row_0/A:col10/1731230854677/Put/seqid=0 2024-11-10T09:27:34,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742303_1479 (size=9857) 2024-11-10T09:27:34,708 INFO [Thread-1852 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1852. 2024-11-10T09:27:34,708 DEBUG [Thread-1852 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:34,708 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:34,709 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:34,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:34,868 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/7eb725e99316437a9d15a3ba173baa6d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/7eb725e99316437a9d15a3ba173baa6d 2024-11-10T09:27:34,868 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/c11c51d93fdc46509eddb6b6b66c675e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/c11c51d93fdc46509eddb6b6b66c675e 2024-11-10T09:27:34,872 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/B of 41bd7632cda810f44aae6fb0260d0e2d into 7eb725e99316437a9d15a3ba173baa6d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:34,872 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:34,872 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/A of 41bd7632cda810f44aae6fb0260d0e2d into c11c51d93fdc46509eddb6b6b66c675e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:34,872 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/B, priority=12, startTime=1731230854445; duration=0sec 2024-11-10T09:27:34,872 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:34,872 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/A, priority=12, startTime=1731230854445; duration=0sec 2024-11-10T09:27:34,872 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:34,872 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:B 2024-11-10T09:27:34,872 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T09:27:34,872 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:34,872 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:A 2024-11-10T09:27:34,874 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T09:27:34,874 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): 41bd7632cda810f44aae6fb0260d0e2d/C is initiating minor compaction (all files) 2024-11-10T09:27:34,874 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41bd7632cda810f44aae6fb0260d0e2d/C in TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:34,874 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c3a3c319fd949bea8ece0987241e81a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/20df6056dbf94a8fa873c63070119a37, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ca6e5fb2e89344069839a6a2251659fb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ff0f4926304f46d7be38ae9657b5baf8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp, totalSize=48.5 K 2024-11-10T09:27:34,874 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c3a3c319fd949bea8ece0987241e81a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731230849938 2024-11-10T09:27:34,874 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 20df6056dbf94a8fa873c63070119a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731230850073 2024-11-10T09:27:34,875 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ca6e5fb2e89344069839a6a2251659fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731230851249 2024-11-10T09:27:34,875 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ff0f4926304f46d7be38ae9657b5baf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731230852028 2024-11-10T09:27:34,882 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41bd7632cda810f44aae6fb0260d0e2d#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:34,883 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/a2bcaf2e82204f7cbb280a3e96408b7f is 50, key is test_row_0/C:col10/1731230852034/Put/seqid=0 2024-11-10T09:27:34,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742304_1480 (size=13085) 2024-11-10T09:27:35,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/53e3bfdae3e64355a5ebbddb11c229ef 2024-11-10T09:27:35,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/695b257ed1ae4022972aee7ab8167aad is 50, key is test_row_0/B:col10/1731230854677/Put/seqid=0 2024-11-10T09:27:35,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742305_1481 (size=9857) 2024-11-10T09:27:35,291 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/a2bcaf2e82204f7cbb280a3e96408b7f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/a2bcaf2e82204f7cbb280a3e96408b7f 2024-11-10T09:27:35,297 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41bd7632cda810f44aae6fb0260d0e2d/C of 41bd7632cda810f44aae6fb0260d0e2d into a2bcaf2e82204f7cbb280a3e96408b7f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:35,297 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:35,297 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., storeName=41bd7632cda810f44aae6fb0260d0e2d/C, priority=12, startTime=1731230854445; duration=0sec 2024-11-10T09:27:35,297 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:35,297 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41bd7632cda810f44aae6fb0260d0e2d:C 2024-11-10T09:27:35,377 INFO [Thread-1850 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1850. 2024-11-10T09:27:35,377 DEBUG [Thread-1850 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:35,377 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:35,377 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:35,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:35,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/695b257ed1ae4022972aee7ab8167aad 2024-11-10T09:27:35,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/0591229b74a840c789a89ea2ce853781 is 50, key is test_row_0/C:col10/1731230854677/Put/seqid=0 2024-11-10T09:27:35,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742306_1482 (size=9857) 2024-11-10T09:27:35,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/0591229b74a840c789a89ea2ce853781 2024-11-10T09:27:35,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/53e3bfdae3e64355a5ebbddb11c229ef as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/53e3bfdae3e64355a5ebbddb11c229ef 2024-11-10T09:27:35,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/53e3bfdae3e64355a5ebbddb11c229ef, entries=100, sequenceid=332, filesize=9.6 K 2024-11-10T09:27:35,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/695b257ed1ae4022972aee7ab8167aad as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/695b257ed1ae4022972aee7ab8167aad 2024-11-10T09:27:35,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/695b257ed1ae4022972aee7ab8167aad, entries=100, sequenceid=332, filesize=9.6 K 2024-11-10T09:27:35,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/0591229b74a840c789a89ea2ce853781 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/0591229b74a840c789a89ea2ce853781 2024-11-10T09:27:35,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/0591229b74a840c789a89ea2ce853781, entries=100, sequenceid=332, filesize=9.6 K 2024-11-10T09:27:35,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=13.42 KB/13740 for 41bd7632cda810f44aae6fb0260d0e2d in 1268ms, sequenceid=332, compaction requested=false 2024-11-10T09:27:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41bd7632cda810f44aae6fb0260d0e2d: 2024-11-10T09:27:40,897 INFO [Thread-1858 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1858. 2024-11-10T09:27:40,897 DEBUG [Thread-1858 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:27:40,897 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:40,897 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:40,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:40,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1839 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5516 rows 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1850 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5546 rows 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1839 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5510 rows 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1849 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5545 rows 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1850 2024-11-10T09:27:40,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5549 rows 2024-11-10T09:27:40,898 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:27:40,898 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:27:40,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:40,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:40,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:27:40,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:27:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:27:40,902 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230860902"}]},"ts":"1731230860902"} 2024-11-10T09:27:40,903 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:27:40,903 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:27:40,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:27:40,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, UNASSIGN}] 2024-11-10T09:27:40,907 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, UNASSIGN 2024-11-10T09:27:40,908 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=41bd7632cda810f44aae6fb0260d0e2d, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:40,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, UNASSIGN because future has completed 2024-11-10T09:27:40,909 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:27:40,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:27:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:27:41,062 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:41,062 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:27:41,063 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 41bd7632cda810f44aae6fb0260d0e2d, disabling compactions & flushes 2024-11-10T09:27:41,063 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:41,063 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:41,063 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. after waiting 0 ms 2024-11-10T09:27:41,063 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:41,063 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 41bd7632cda810f44aae6fb0260d0e2d 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-10T09:27:41,070 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/4fa4e55b9c664e58816220045d593fc1 is 50, key is test_row_0/A:col10/1731230854706/Put/seqid=0 2024-11-10T09:27:41,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742307_1483 (size=9857) 2024-11-10T09:27:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:27:41,477 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/4fa4e55b9c664e58816220045d593fc1 2024-11-10T09:27:41,490 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/2c6a816ada8c4255a04e700c62eda0c3 is 50, key is test_row_0/B:col10/1731230854706/Put/seqid=0 2024-11-10T09:27:41,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742308_1484 (size=9857) 2024-11-10T09:27:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:27:41,895 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/2c6a816ada8c4255a04e700c62eda0c3 2024-11-10T09:27:41,907 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/23f6d41416e349869864565a5cd23c75 is 50, key is test_row_0/C:col10/1731230854706/Put/seqid=0 2024-11-10T09:27:41,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742309_1485 (size=9857) 2024-11-10T09:27:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:27:42,313 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/23f6d41416e349869864565a5cd23c75 2024-11-10T09:27:42,323 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/A/4fa4e55b9c664e58816220045d593fc1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/4fa4e55b9c664e58816220045d593fc1 2024-11-10T09:27:42,326 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/4fa4e55b9c664e58816220045d593fc1, entries=100, sequenceid=341, filesize=9.6 K 2024-11-10T09:27:42,327 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/B/2c6a816ada8c4255a04e700c62eda0c3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2c6a816ada8c4255a04e700c62eda0c3 2024-11-10T09:27:42,330 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2c6a816ada8c4255a04e700c62eda0c3, entries=100, sequenceid=341, filesize=9.6 K 2024-11-10T09:27:42,331 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/.tmp/C/23f6d41416e349869864565a5cd23c75 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/23f6d41416e349869864565a5cd23c75 2024-11-10T09:27:42,334 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/23f6d41416e349869864565a5cd23c75, entries=100, sequenceid=341, filesize=9.6 K 2024-11-10T09:27:42,335 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 41bd7632cda810f44aae6fb0260d0e2d in 1272ms, sequenceid=341, compaction requested=true 2024-11-10T09:27:42,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/85d4a46f607b492692113b2d72c0ea10, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/19f7524415d84b80be328d6155a82071, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/31170776382040709c448f1b154350ed, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/adb54addec3641d98d1d006325f9958a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2ed5b98ccada45bab7d7e0abdc98d7a0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/fecb6a72a4494c26838d5b7f331c8d42, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2dd737f964894a90b3e041b09ddebdd4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d8661232e27d4aaab41e52ae99a90dfd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/75574027412e48ec8666af7fb73ca504, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/3a0dd8f46b9d40a898629cfbe5e49b09, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/13abd25ede5d4800a792e1bcd535a6ca, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/efe0d198145f4316ba12f0c2c2560231, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/e7faa3a7a0a2424294a0503b3c7ff60d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/af23c7a257494fc4baeedcd3c018bb5a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/86a8e9bb09c24455b0a4a1bf4b349b83, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/6a5e2f8d51f44a34aac60fb53c9bf3e2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/0f71ec76644949b38413c9d30e7d268c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9bd99b5460944d81968afa753f0de91b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d18e518def4e45b18369784244063600, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/f5e3adc7680f4e4c8a41419643f5699f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9eb4d55fdc4c4d269160a0cc656d2a02, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d43deabad62449a09413f6211d84324f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8] to archive 2024-11-10T09:27:42,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:27:42,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/85d4a46f607b492692113b2d72c0ea10 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/85d4a46f607b492692113b2d72c0ea10 2024-11-10T09:27:42,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/19f7524415d84b80be328d6155a82071 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/19f7524415d84b80be328d6155a82071 2024-11-10T09:27:42,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/31170776382040709c448f1b154350ed to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/31170776382040709c448f1b154350ed 2024-11-10T09:27:42,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/adb54addec3641d98d1d006325f9958a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/adb54addec3641d98d1d006325f9958a 2024-11-10T09:27:42,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2ed5b98ccada45bab7d7e0abdc98d7a0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2ed5b98ccada45bab7d7e0abdc98d7a0 2024-11-10T09:27:42,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/fecb6a72a4494c26838d5b7f331c8d42 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/fecb6a72a4494c26838d5b7f331c8d42 2024-11-10T09:27:42,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2dd737f964894a90b3e041b09ddebdd4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/2dd737f964894a90b3e041b09ddebdd4 2024-11-10T09:27:42,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d8661232e27d4aaab41e52ae99a90dfd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d8661232e27d4aaab41e52ae99a90dfd 2024-11-10T09:27:42,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/75574027412e48ec8666af7fb73ca504 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/75574027412e48ec8666af7fb73ca504 2024-11-10T09:27:42,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/3a0dd8f46b9d40a898629cfbe5e49b09 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/3a0dd8f46b9d40a898629cfbe5e49b09 2024-11-10T09:27:42,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/13abd25ede5d4800a792e1bcd535a6ca to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/13abd25ede5d4800a792e1bcd535a6ca 2024-11-10T09:27:42,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/efe0d198145f4316ba12f0c2c2560231 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/efe0d198145f4316ba12f0c2c2560231 2024-11-10T09:27:42,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/e7faa3a7a0a2424294a0503b3c7ff60d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/e7faa3a7a0a2424294a0503b3c7ff60d 2024-11-10T09:27:42,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/af23c7a257494fc4baeedcd3c018bb5a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/af23c7a257494fc4baeedcd3c018bb5a 2024-11-10T09:27:42,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/86a8e9bb09c24455b0a4a1bf4b349b83 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/86a8e9bb09c24455b0a4a1bf4b349b83 2024-11-10T09:27:42,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/6a5e2f8d51f44a34aac60fb53c9bf3e2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/6a5e2f8d51f44a34aac60fb53c9bf3e2 2024-11-10T09:27:42,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/0f71ec76644949b38413c9d30e7d268c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/0f71ec76644949b38413c9d30e7d268c 2024-11-10T09:27:42,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9bd99b5460944d81968afa753f0de91b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9bd99b5460944d81968afa753f0de91b 2024-11-10T09:27:42,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d18e518def4e45b18369784244063600 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d18e518def4e45b18369784244063600 2024-11-10T09:27:42,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/f5e3adc7680f4e4c8a41419643f5699f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/f5e3adc7680f4e4c8a41419643f5699f 2024-11-10T09:27:42,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9eb4d55fdc4c4d269160a0cc656d2a02 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/9eb4d55fdc4c4d269160a0cc656d2a02 2024-11-10T09:27:42,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d43deabad62449a09413f6211d84324f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/d43deabad62449a09413f6211d84324f 2024-11-10T09:27:42,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/a60dcfb6b0fd4f68acf20cb2ecc0c9c8 2024-11-10T09:27:42,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ad37bbf4f1a242ff913bc60659a80f98, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/bff757222831437da60232c6318908ff, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d24c3c1892f043bcaedcca05d6b9d347, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ea797b40658a4315b0a188a7a8104d0e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e8f25fecf3e643c1b3a6849faf8796be, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/9162e50cfa0a4190918d61c97e3932e8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/4f8c1799ae7d45269275ec93da13fea5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/f1e9e7953b184f52895177355fbeb418, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/02c12c03f6e5438a95fb735af9e41cb6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/36efe123588042688378bc971f15c53e, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/696ca29bc7034a768b87fab150bdd19a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/774dd133f7744c05a1b8304d3255b47b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/c0955c40e91e42d79b8ca9cc10994858, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/acf14d67d0504261974276e42cd1ae60, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/a22e56e4a4c54beb87005d3d5a3068d3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d07bfaa5e2e24a5fbe5b01b9eec65471, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/eee0b71a8d4b448f9d9a3b717a11a521, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ee5a16e97f0940ac9dd43e73c5bd08d8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/19bbb63027424693907bdc8ca2f6f6e8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e5ec32470c7646ff890eb4f9f1d1b0ed, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/08b86767127a4484a858c2aedfaf9173, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/161476c5d69c4e2aa2558047528021f1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2d121c76c0ac491280fd165241b7c590] to archive 2024-11-10T09:27:42,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:27:42,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ad37bbf4f1a242ff913bc60659a80f98 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ad37bbf4f1a242ff913bc60659a80f98 2024-11-10T09:27:42,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/bff757222831437da60232c6318908ff to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/bff757222831437da60232c6318908ff 2024-11-10T09:27:42,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d24c3c1892f043bcaedcca05d6b9d347 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d24c3c1892f043bcaedcca05d6b9d347 2024-11-10T09:27:42,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ea797b40658a4315b0a188a7a8104d0e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ea797b40658a4315b0a188a7a8104d0e 2024-11-10T09:27:42,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e8f25fecf3e643c1b3a6849faf8796be to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e8f25fecf3e643c1b3a6849faf8796be 2024-11-10T09:27:42,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/9162e50cfa0a4190918d61c97e3932e8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/9162e50cfa0a4190918d61c97e3932e8 2024-11-10T09:27:42,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/4f8c1799ae7d45269275ec93da13fea5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/4f8c1799ae7d45269275ec93da13fea5 2024-11-10T09:27:42,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/f1e9e7953b184f52895177355fbeb418 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/f1e9e7953b184f52895177355fbeb418 2024-11-10T09:27:42,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/02c12c03f6e5438a95fb735af9e41cb6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/02c12c03f6e5438a95fb735af9e41cb6 2024-11-10T09:27:42,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/36efe123588042688378bc971f15c53e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/36efe123588042688378bc971f15c53e 2024-11-10T09:27:42,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/696ca29bc7034a768b87fab150bdd19a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/696ca29bc7034a768b87fab150bdd19a 2024-11-10T09:27:42,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/774dd133f7744c05a1b8304d3255b47b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/774dd133f7744c05a1b8304d3255b47b 2024-11-10T09:27:42,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/c0955c40e91e42d79b8ca9cc10994858 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/c0955c40e91e42d79b8ca9cc10994858 2024-11-10T09:27:42,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/acf14d67d0504261974276e42cd1ae60 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/acf14d67d0504261974276e42cd1ae60 2024-11-10T09:27:42,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/a22e56e4a4c54beb87005d3d5a3068d3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/a22e56e4a4c54beb87005d3d5a3068d3 2024-11-10T09:27:42,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d07bfaa5e2e24a5fbe5b01b9eec65471 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/d07bfaa5e2e24a5fbe5b01b9eec65471 2024-11-10T09:27:42,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/eee0b71a8d4b448f9d9a3b717a11a521 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/eee0b71a8d4b448f9d9a3b717a11a521 2024-11-10T09:27:42,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ee5a16e97f0940ac9dd43e73c5bd08d8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/ee5a16e97f0940ac9dd43e73c5bd08d8 2024-11-10T09:27:42,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/19bbb63027424693907bdc8ca2f6f6e8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/19bbb63027424693907bdc8ca2f6f6e8 2024-11-10T09:27:42,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e5ec32470c7646ff890eb4f9f1d1b0ed to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/e5ec32470c7646ff890eb4f9f1d1b0ed 2024-11-10T09:27:42,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/08b86767127a4484a858c2aedfaf9173 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/08b86767127a4484a858c2aedfaf9173 2024-11-10T09:27:42,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/161476c5d69c4e2aa2558047528021f1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/161476c5d69c4e2aa2558047528021f1 2024-11-10T09:27:42,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2d121c76c0ac491280fd165241b7c590 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2d121c76c0ac491280fd165241b7c590 2024-11-10T09:27:42,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/f7352ec59d1f450e9a60746d646c44a3, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/527909507d2a482894799d01ea2687b4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c56589245b64381ad7c121a70b242ef, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/b4f14b9616094856965e386b8e26d964, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/49637b5b2dbc45ddb0c0f73246e5b93d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2d25ec3fbb1341bdae431fb223f15658, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/d2b92c270d504ca8ab5b5168ddb9500c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7813542b42444d739c4d688d82f17417, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/fe1a374642ae447791d4b5da81573d4a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5a881583bf5246fab588a3b0bcabbbb1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/157b09c6961d4f4f94d3fb840adb7632, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/90757160c4914c5eb0e4745aea41e3c0, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/e237cd15cde241378fce05a1a0155dfd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aae9a47315764fbebc4a2d8d4a5be08b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/66dacd561a1845618b24ed6f2adcdf19, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2b5189c6ca8445d3987110ec9abe7ffd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5c4ce9d5675e4c16954d86d228964aa7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aa2e5823ab9e4ca6a04c3f54a985af7a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c3a3c319fd949bea8ece0987241e81a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ee5f442709754457a6d4545939578e4d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/20df6056dbf94a8fa873c63070119a37, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ca6e5fb2e89344069839a6a2251659fb, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ff0f4926304f46d7be38ae9657b5baf8] to archive 2024-11-10T09:27:42,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:27:42,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/f7352ec59d1f450e9a60746d646c44a3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/f7352ec59d1f450e9a60746d646c44a3 2024-11-10T09:27:42,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/527909507d2a482894799d01ea2687b4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/527909507d2a482894799d01ea2687b4 2024-11-10T09:27:42,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c56589245b64381ad7c121a70b242ef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c56589245b64381ad7c121a70b242ef 2024-11-10T09:27:42,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/b4f14b9616094856965e386b8e26d964 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/b4f14b9616094856965e386b8e26d964 2024-11-10T09:27:42,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/49637b5b2dbc45ddb0c0f73246e5b93d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/49637b5b2dbc45ddb0c0f73246e5b93d 2024-11-10T09:27:42,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2d25ec3fbb1341bdae431fb223f15658 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2d25ec3fbb1341bdae431fb223f15658 2024-11-10T09:27:42,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/d2b92c270d504ca8ab5b5168ddb9500c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/d2b92c270d504ca8ab5b5168ddb9500c 2024-11-10T09:27:42,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7813542b42444d739c4d688d82f17417 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7813542b42444d739c4d688d82f17417 2024-11-10T09:27:42,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/fe1a374642ae447791d4b5da81573d4a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/fe1a374642ae447791d4b5da81573d4a 2024-11-10T09:27:42,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5a881583bf5246fab588a3b0bcabbbb1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5a881583bf5246fab588a3b0bcabbbb1 2024-11-10T09:27:42,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/157b09c6961d4f4f94d3fb840adb7632 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/157b09c6961d4f4f94d3fb840adb7632 2024-11-10T09:27:42,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/90757160c4914c5eb0e4745aea41e3c0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/90757160c4914c5eb0e4745aea41e3c0 2024-11-10T09:27:42,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/e237cd15cde241378fce05a1a0155dfd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/e237cd15cde241378fce05a1a0155dfd 2024-11-10T09:27:42,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aae9a47315764fbebc4a2d8d4a5be08b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aae9a47315764fbebc4a2d8d4a5be08b 2024-11-10T09:27:42,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/66dacd561a1845618b24ed6f2adcdf19 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/66dacd561a1845618b24ed6f2adcdf19 2024-11-10T09:27:42,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2b5189c6ca8445d3987110ec9abe7ffd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/2b5189c6ca8445d3987110ec9abe7ffd 2024-11-10T09:27:42,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5c4ce9d5675e4c16954d86d228964aa7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/5c4ce9d5675e4c16954d86d228964aa7 2024-11-10T09:27:42,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aa2e5823ab9e4ca6a04c3f54a985af7a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/aa2e5823ab9e4ca6a04c3f54a985af7a 2024-11-10T09:27:42,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c3a3c319fd949bea8ece0987241e81a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/7c3a3c319fd949bea8ece0987241e81a 2024-11-10T09:27:42,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ee5f442709754457a6d4545939578e4d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ee5f442709754457a6d4545939578e4d 2024-11-10T09:27:42,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/20df6056dbf94a8fa873c63070119a37 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/20df6056dbf94a8fa873c63070119a37 2024-11-10T09:27:42,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ca6e5fb2e89344069839a6a2251659fb to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ca6e5fb2e89344069839a6a2251659fb 2024-11-10T09:27:42,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ff0f4926304f46d7be38ae9657b5baf8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/ff0f4926304f46d7be38ae9657b5baf8 2024-11-10T09:27:42,418 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/recovered.edits/344.seqid, newMaxSeqId=344, maxSeqId=1 2024-11-10T09:27:42,418 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. 2024-11-10T09:27:42,418 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 41bd7632cda810f44aae6fb0260d0e2d: Waiting for close lock at 1731230861063Running coprocessor pre-close hooks at 1731230861063Disabling compacts and flushes for region at 1731230861063Disabling writes for close at 1731230861063Obtaining lock to block concurrent updates at 1731230861063Preparing flush snapshotting stores in 41bd7632cda810f44aae6fb0260d0e2d at 1731230861063Finished memstore snapshotting TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731230861064 (+1 ms)Flushing stores of TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d. at 1731230861065 (+1 ms)Flushing 41bd7632cda810f44aae6fb0260d0e2d/A: creating writer at 1731230861065Flushing 41bd7632cda810f44aae6fb0260d0e2d/A: appending metadata at 1731230861070 (+5 ms)Flushing 41bd7632cda810f44aae6fb0260d0e2d/A: closing flushed file at 1731230861070Flushing 41bd7632cda810f44aae6fb0260d0e2d/B: creating writer at 1731230861487 (+417 ms)Flushing 41bd7632cda810f44aae6fb0260d0e2d/B: appending metadata at 1731230861490 (+3 ms)Flushing 41bd7632cda810f44aae6fb0260d0e2d/B: closing flushed file at 1731230861490Flushing 41bd7632cda810f44aae6fb0260d0e2d/C: creating writer at 1731230861904 (+414 ms)Flushing 41bd7632cda810f44aae6fb0260d0e2d/C: appending metadata at 1731230861907 (+3 ms)Flushing 41bd7632cda810f44aae6fb0260d0e2d/C: closing flushed file at 1731230861907Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1708ca3a: reopening flushed file at 1731230862322 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4249f164: reopening flushed file at 1731230862326 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c66dd4e: reopening flushed file at 1731230862331 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 41bd7632cda810f44aae6fb0260d0e2d in 1272ms, sequenceid=341, compaction requested=true at 1731230862335 (+4 ms)Writing region close event to WAL at 1731230862415 (+80 ms)Running coprocessor post-close hooks at 1731230862418 (+3 ms)Closed at 1731230862418 2024-11-10T09:27:42,419 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:42,420 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=41bd7632cda810f44aae6fb0260d0e2d, regionState=CLOSED 2024-11-10T09:27:42,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:27:42,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-10T09:27:42,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 41bd7632cda810f44aae6fb0260d0e2d, server=802346a334f5,32883,1731230704436 in 1.5130 sec 2024-11-10T09:27:42,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-10T09:27:42,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41bd7632cda810f44aae6fb0260d0e2d, UNASSIGN in 1.5170 sec 2024-11-10T09:27:42,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T09:27:42,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-10T09:27:42,427 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230862427"}]},"ts":"1731230862427"} 2024-11-10T09:27:42,428 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:27:42,428 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:27:42,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5300 sec 2024-11-10T09:27:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-10T09:27:43,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:43,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:27:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:43,061 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T09:27:43,063 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:43,067 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:43,071 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/recovered.edits] 2024-11-10T09:27:43,074 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/4fa4e55b9c664e58816220045d593fc1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/4fa4e55b9c664e58816220045d593fc1 2024-11-10T09:27:43,075 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/53e3bfdae3e64355a5ebbddb11c229ef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/53e3bfdae3e64355a5ebbddb11c229ef 2024-11-10T09:27:43,076 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/c11c51d93fdc46509eddb6b6b66c675e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/A/c11c51d93fdc46509eddb6b6b66c675e 2024-11-10T09:27:43,077 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2c6a816ada8c4255a04e700c62eda0c3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/2c6a816ada8c4255a04e700c62eda0c3 2024-11-10T09:27:43,078 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/695b257ed1ae4022972aee7ab8167aad to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/695b257ed1ae4022972aee7ab8167aad 2024-11-10T09:27:43,079 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/7eb725e99316437a9d15a3ba173baa6d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/B/7eb725e99316437a9d15a3ba173baa6d 2024-11-10T09:27:43,081 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/0591229b74a840c789a89ea2ce853781 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/0591229b74a840c789a89ea2ce853781 2024-11-10T09:27:43,082 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/23f6d41416e349869864565a5cd23c75 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/23f6d41416e349869864565a5cd23c75 2024-11-10T09:27:43,083 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/a2bcaf2e82204f7cbb280a3e96408b7f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/C/a2bcaf2e82204f7cbb280a3e96408b7f 2024-11-10T09:27:43,085 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/recovered.edits/344.seqid to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d/recovered.edits/344.seqid 2024-11-10T09:27:43,085 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/41bd7632cda810f44aae6fb0260d0e2d 2024-11-10T09:27:43,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:27:43,087 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:43,089 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:27:43,090 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:27:43,091 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:43,091 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:27:43,092 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230863091"}]},"ts":"9223372036854775807"} 2024-11-10T09:27:43,093 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:27:43,093 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 41bd7632cda810f44aae6fb0260d0e2d, NAME => 'TestAcidGuarantees,,1731230831266.41bd7632cda810f44aae6fb0260d0e2d.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:27:43,093 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:27:43,094 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230863093"}]},"ts":"9223372036854775807"} 2024-11-10T09:27:43,095 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:27:43,096 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:43,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-10T09:27:43,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-10T09:27:43,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:27:43,171 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:43,181 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 243), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=350 (was 445), ProcessCount=11 (was 11), AvailableMemoryMB=6832 (was 6850) 2024-11-10T09:27:43,188 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=350, ProcessCount=11, AvailableMemoryMB=6831 2024-11-10T09:27:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:27:43,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:27:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:43,190 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T09:27:43,191 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:43,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-10T09:27:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:27:43,191 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T09:27:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742310_1486 (size=959) 2024-11-10T09:27:43,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:27:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:27:43,602 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6 2024-11-10T09:27:43,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742311_1487 (size=53) 2024-11-10T09:27:43,772 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:27:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:27:44,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:27:44,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dedf9887fe3cb3c93b7ecf99bd810cef, disabling compactions & flushes 2024-11-10T09:27:44,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:44,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:44,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. after waiting 0 ms 2024-11-10T09:27:44,013 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:44,013 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:44,014 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dedf9887fe3cb3c93b7ecf99bd810cef: Waiting for close lock at 1731230864013Disabling compacts and flushes for region at 1731230864013Disabling writes for close at 1731230864013Writing region close event to WAL at 1731230864013Closed at 1731230864013 2024-11-10T09:27:44,017 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T09:27:44,017 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731230864017"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731230864017"}]},"ts":"1731230864017"} 2024-11-10T09:27:44,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T09:27:44,023 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T09:27:44,024 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230864023"}]},"ts":"1731230864023"} 2024-11-10T09:27:44,027 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T09:27:44,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, ASSIGN}] 2024-11-10T09:27:44,029 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, ASSIGN 2024-11-10T09:27:44,030 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, ASSIGN; state=OFFLINE, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=false 2024-11-10T09:27:44,181 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:44,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, ASSIGN because future has completed 2024-11-10T09:27:44,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:27:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:27:44,344 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:44,344 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:27:44,344 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,345 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:27:44,345 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,345 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,347 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,349 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dedf9887fe3cb3c93b7ecf99bd810cef columnFamilyName A 2024-11-10T09:27:44,349 DEBUG [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:44,350 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(327): Store=dedf9887fe3cb3c93b7ecf99bd810cef/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:44,350 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,352 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dedf9887fe3cb3c93b7ecf99bd810cef columnFamilyName B 2024-11-10T09:27:44,352 DEBUG [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:44,353 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(327): Store=dedf9887fe3cb3c93b7ecf99bd810cef/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:44,353 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,355 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dedf9887fe3cb3c93b7ecf99bd810cef columnFamilyName C 2024-11-10T09:27:44,355 DEBUG [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:44,355 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(327): Store=dedf9887fe3cb3c93b7ecf99bd810cef/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:44,356 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,357 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,358 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,360 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,360 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,361 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:27:44,362 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,365 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T09:27:44,366 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened dedf9887fe3cb3c93b7ecf99bd810cef; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67353661, jitterRate=0.0036477595567703247}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:27:44,366 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:44,367 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for dedf9887fe3cb3c93b7ecf99bd810cef: Running coprocessor pre-open hook at 1731230864345Writing region info on filesystem at 1731230864345Initializing all the Stores at 1731230864346 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230864346Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230864347 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230864347Cleaning up temporary data from old regions at 1731230864360 (+13 ms)Running coprocessor post-open hooks at 1731230864366 (+6 ms)Region opened successfully at 1731230864367 (+1 ms) 2024-11-10T09:27:44,368 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., pid=146, masterSystemTime=1731230864340 2024-11-10T09:27:44,370 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:44,370 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:44,371 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=OPEN, openSeqNum=2, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:44,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:27:44,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-10T09:27:44,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 in 189 msec 2024-11-10T09:27:44,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-10T09:27:44,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, ASSIGN in 350 msec 2024-11-10T09:27:44,381 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T09:27:44,381 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230864381"}]},"ts":"1731230864381"} 2024-11-10T09:27:44,383 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T09:27:44,384 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T09:27:44,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-10T09:27:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T09:27:45,337 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:45,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6f989f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:45,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:45,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:45,343 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:45,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e8f8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:45,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:45,348 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:45,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@868bc74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:45,350 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:45,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:45,353 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T09:27:45,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 802346a334f5,38235,1731230703274 2024-11-10T09:27:45,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ad9e2d6 2024-11-10T09:27:45,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T09:27:45,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T09:27:45,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T09:27:45,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T09:27:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T09:27:45,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742312_1488 (size=995) 2024-11-10T09:27:45,772 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-10T09:27:45,772 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-10T09:27:45,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:27:45,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, REOPEN/MOVE}] 2024-11-10T09:27:45,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, REOPEN/MOVE 2024-11-10T09:27:45,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:45,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, REOPEN/MOVE because future has completed 2024-11-10T09:27:45,784 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:27:45,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:27:45,938 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:45,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:27:45,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing dedf9887fe3cb3c93b7ecf99bd810cef, disabling compactions & flushes 2024-11-10T09:27:45,938 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:45,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:45,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. after waiting 0 ms 2024-11-10T09:27:45,938 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:45,948 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T09:27:45,948 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:45,948 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for dedf9887fe3cb3c93b7ecf99bd810cef: Waiting for close lock at 1731230865938Running coprocessor pre-close hooks at 1731230865938Disabling compacts and flushes for region at 1731230865938Disabling writes for close at 1731230865938Writing region close event to WAL at 1731230865940 (+2 ms)Running coprocessor post-close hooks at 1731230865948 (+8 ms)Closed at 1731230865948 2024-11-10T09:27:45,948 WARN [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: dedf9887fe3cb3c93b7ecf99bd810cef to self. 2024-11-10T09:27:45,950 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:45,951 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=CLOSED 2024-11-10T09:27:45,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:27:45,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-10T09:27:45,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 in 169 msec 2024-11-10T09:27:45,955 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, REOPEN/MOVE; state=CLOSED, location=802346a334f5,32883,1731230704436; forceNewPlan=false, retain=true 2024-11-10T09:27:46,107 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=OPENING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, REOPEN/MOVE because future has completed 2024-11-10T09:27:46,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:27:46,269 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,270 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} 2024-11-10T09:27:46,270 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,270 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T09:27:46,271 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,271 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,273 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,275 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dedf9887fe3cb3c93b7ecf99bd810cef columnFamilyName A 2024-11-10T09:27:46,278 DEBUG [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:46,279 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(327): Store=dedf9887fe3cb3c93b7ecf99bd810cef/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:46,279 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,280 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dedf9887fe3cb3c93b7ecf99bd810cef columnFamilyName B 2024-11-10T09:27:46,280 DEBUG [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:46,281 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(327): Store=dedf9887fe3cb3c93b7ecf99bd810cef/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:46,281 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,282 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dedf9887fe3cb3c93b7ecf99bd810cef columnFamilyName C 2024-11-10T09:27:46,283 DEBUG [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:46,283 INFO [StoreOpener-dedf9887fe3cb3c93b7ecf99bd810cef-1 {}] regionserver.HStore(327): Store=dedf9887fe3cb3c93b7ecf99bd810cef/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T09:27:46,284 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,285 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,287 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,288 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,289 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,289 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T09:27:46,291 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,292 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened dedf9887fe3cb3c93b7ecf99bd810cef; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73003254, jitterRate=0.08783325552940369}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T09:27:46,292 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,293 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for dedf9887fe3cb3c93b7ecf99bd810cef: Running coprocessor pre-open hook at 1731230866271Writing region info on filesystem at 1731230866271Initializing all the Stores at 1731230866272 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230866273 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230866273Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731230866273Cleaning up temporary data from old regions at 1731230866289 (+16 ms)Running coprocessor post-open hooks at 1731230866292 (+3 ms)Region opened successfully at 1731230866293 (+1 ms) 2024-11-10T09:27:46,294 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., pid=151, masterSystemTime=1731230866264 2024-11-10T09:27:46,297 DEBUG [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,297 INFO [RS_OPEN_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,297 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=OPEN, openSeqNum=5, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:27:46,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-10T09:27:46,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 in 190 msec 2024-11-10T09:27:46,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T09:27:46,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, REOPEN/MOVE in 523 msec 2024-11-10T09:27:46,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-10T09:27:46,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-10T09:27:46,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-10T09:27:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-10T09:27:46,310 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:46,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a209286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69480472, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,312 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c23ea2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fa0b51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,314 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,315 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c78a629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1130c997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,317 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bce0531, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,317 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da20cae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29acc1f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,319 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246b99c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,320 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286c2aba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b691ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,321 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a928da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ff2a6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3420b4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,324 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670ebf88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,325 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac81ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1b2f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,327 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37173d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,328 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@790a26d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1102b71d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,330 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2d3b2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,331 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65751e0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ff4806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,333 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769f4e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a805869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8677bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 802346a334f5,38235,-1 for getting cluster id 2024-11-10T09:27:46,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T09:27:46,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31bb2194-f5a5-4093-a2ad-d78528cf56b8' 2024-11-10T09:27:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T09:27:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31bb2194-f5a5-4093-a2ad-d78528cf56b8" 2024-11-10T09:27:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f39e29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [802346a334f5,38235,-1] 2024-11-10T09:27:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T09:27:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:27:46,336 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T09:27:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f937a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T09:27:46,337 DEBUG [Thread-2204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,338 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,338 DEBUG [Thread-2208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,339 DEBUG [Thread-2221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,339 DEBUG [Thread-2217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,339 DEBUG [Thread-2223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,340 DEBUG [Thread-2219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,343 DEBUG [Thread-2206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,349 DEBUG [Thread-2215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,350 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:46,350 DEBUG [Thread-2215 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,350 DEBUG [Thread-2215 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-10T09:27:46,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:27:46,354 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:46,355 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:46,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:27:46,355 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:46,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:46,359 DEBUG [Thread-2210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T09:27:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=802346a334f5,32883,1731230704436, seqNum=-1] 2024-11-10T09:27:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T09:27:46,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T09:27:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5] 2024-11-10T09:27:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230926384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230926385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230926384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107dfe8722414f4d37a92b7907f301f6cb_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230866348/Put/seqid=0 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230926386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230926389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742313_1489 (size=12154) 2024-11-10T09:27:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:27:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230926496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230926496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230926496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230926496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230926505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:46,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:46,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:46,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:46,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:27:46,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230926706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230926706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230926706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230926706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230926715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:46,797 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:46,799 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107dfe8722414f4d37a92b7907f301f6cb_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107dfe8722414f4d37a92b7907f301f6cb_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:46,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/e042482d642d4171bfd135fc17e07d27, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:46,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/e042482d642d4171bfd135fc17e07d27 is 175, key is test_row_0/A:col10/1731230866348/Put/seqid=0 2024-11-10T09:27:46,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742314_1490 (size=30955) 2024-11-10T09:27:46,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:46,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:46,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:46,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:27:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230927016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230927016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230927016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230927016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230927025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:47,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,204 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/e042482d642d4171bfd135fc17e07d27 2024-11-10T09:27:47,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/961a7a287c9f47fe82c5b7c96d189f4b is 50, key is test_row_0/B:col10/1731230866348/Put/seqid=0 2024-11-10T09:27:47,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742315_1491 (size=12001) 2024-11-10T09:27:47,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:27:47,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230927526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230927526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230927526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230927526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230927535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:47,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:47,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/961a7a287c9f47fe82c5b7c96d189f4b 2024-11-10T09:27:47,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/d2b77755c8be47599fb473a408881afc is 50, key is test_row_0/C:col10/1731230866348/Put/seqid=0 2024-11-10T09:27:47,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742316_1492 (size=12001) 2024-11-10T09:27:47,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/d2b77755c8be47599fb473a408881afc 2024-11-10T09:27:47,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/e042482d642d4171bfd135fc17e07d27 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/e042482d642d4171bfd135fc17e07d27 2024-11-10T09:27:47,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/e042482d642d4171bfd135fc17e07d27, entries=150, sequenceid=17, filesize=30.2 K 2024-11-10T09:27:47,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/961a7a287c9f47fe82c5b7c96d189f4b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/961a7a287c9f47fe82c5b7c96d189f4b 2024-11-10T09:27:47,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/961a7a287c9f47fe82c5b7c96d189f4b, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T09:27:47,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/d2b77755c8be47599fb473a408881afc as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d2b77755c8be47599fb473a408881afc 2024-11-10T09:27:47,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d2b77755c8be47599fb473a408881afc, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T09:27:47,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for dedf9887fe3cb3c93b7ecf99bd810cef in 1304ms, sequenceid=17, compaction requested=false 2024-11-10T09:27:47,659 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T09:27:47,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:47,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T09:27:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:47,728 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:27:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f42ee98e68584bae983f3ba5dea0e02e_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230866387/Put/seqid=0 2024-11-10T09:27:47,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742317_1493 (size=12154) 2024-11-10T09:27:47,937 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T09:27:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:48,140 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f42ee98e68584bae983f3ba5dea0e02e_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f42ee98e68584bae983f3ba5dea0e02e_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/908985dbfa814569894cf61656df5ee6, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/908985dbfa814569894cf61656df5ee6 is 175, key is test_row_0/A:col10/1731230866387/Put/seqid=0 2024-11-10T09:27:48,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742318_1494 (size=30955) 2024-11-10T09:27:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:27:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:48,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:48,545 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/908985dbfa814569894cf61656df5ee6 2024-11-10T09:27:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230928547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230928547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/3f15da88c5094620a31c75af898a45ef is 50, key is test_row_0/B:col10/1731230866387/Put/seqid=0 2024-11-10T09:27:48,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230928549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230928549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742319_1495 (size=12001) 2024-11-10T09:27:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230928555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,557 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230928656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230928656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230928656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230928656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230928866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230928866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230928866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230928866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:48,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:48,955 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/3f15da88c5094620a31c75af898a45ef 2024-11-10T09:27:48,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/247a3efb89ee42119d19ebf048dc0fae is 50, key is test_row_0/C:col10/1731230866387/Put/seqid=0 2024-11-10T09:27:48,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742320_1496 (size=12001) 2024-11-10T09:27:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230929175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230929175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230929176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230929176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,370 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/247a3efb89ee42119d19ebf048dc0fae 2024-11-10T09:27:49,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/908985dbfa814569894cf61656df5ee6 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/908985dbfa814569894cf61656df5ee6 2024-11-10T09:27:49,376 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/908985dbfa814569894cf61656df5ee6, entries=150, sequenceid=40, filesize=30.2 K 2024-11-10T09:27:49,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/3f15da88c5094620a31c75af898a45ef as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/3f15da88c5094620a31c75af898a45ef 2024-11-10T09:27:49,379 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/3f15da88c5094620a31c75af898a45ef, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T09:27:49,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/247a3efb89ee42119d19ebf048dc0fae as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/247a3efb89ee42119d19ebf048dc0fae 2024-11-10T09:27:49,382 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/247a3efb89ee42119d19ebf048dc0fae, entries=150, sequenceid=40, filesize=11.7 K 2024-11-10T09:27:49,383 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for dedf9887fe3cb3c93b7ecf99bd810cef in 1655ms, sequenceid=40, compaction requested=false 2024-11-10T09:27:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-10T09:27:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-10T09:27:49,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T09:27:49,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0290 sec 2024-11-10T09:27:49,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0360 sec 2024-11-10T09:27:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:49,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:49,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T09:27:49,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111049a500a0882e492b8e327260448e47cd_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230868548/Put/seqid=0 2024-11-10T09:27:49,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742321_1497 (size=14594) 2024-11-10T09:27:49,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230929711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230929712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230929712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230929713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230929816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230929826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230929826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230929826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230930025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230930036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230930036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230930036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,098 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:50,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111049a500a0882e492b8e327260448e47cd_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111049a500a0882e492b8e327260448e47cd_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:50,102 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/634d62a6ebdb4bbb9d1510c0f474d188, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:50,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/634d62a6ebdb4bbb9d1510c0f474d188 is 175, key is test_row_0/A:col10/1731230868548/Put/seqid=0 2024-11-10T09:27:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742322_1498 (size=39549) 2024-11-10T09:27:50,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230930335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230930345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230930345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230930346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,505 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/634d62a6ebdb4bbb9d1510c0f474d188 2024-11-10T09:27:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T09:27:50,506 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:50,506 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-10T09:27:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:27:50,508 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:50,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:50,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:50,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/2d9c72beab60417980a5c43245c8c645 is 50, key is test_row_0/B:col10/1731230868548/Put/seqid=0 2024-11-10T09:27:50,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742323_1499 (size=12001) 2024-11-10T09:27:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230930575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,578 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:27:50,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T09:27:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:50,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:50,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T09:27:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:50,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:27:50,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230930846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230930856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230930856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230930856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:50,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:50,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/2d9c72beab60417980a5c43245c8c645 2024-11-10T09:27:50,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/0f948ce8e2484cf3a801ef34366adcc4 is 50, key is test_row_0/C:col10/1731230868548/Put/seqid=0 2024-11-10T09:27:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742324_1500 (size=12001) 2024-11-10T09:27:50,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/0f948ce8e2484cf3a801ef34366adcc4 2024-11-10T09:27:50,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/634d62a6ebdb4bbb9d1510c0f474d188 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/634d62a6ebdb4bbb9d1510c0f474d188 2024-11-10T09:27:50,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/634d62a6ebdb4bbb9d1510c0f474d188, entries=200, sequenceid=55, filesize=38.6 K 2024-11-10T09:27:50,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/2d9c72beab60417980a5c43245c8c645 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/2d9c72beab60417980a5c43245c8c645 2024-11-10T09:27:50,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/2d9c72beab60417980a5c43245c8c645, entries=150, sequenceid=55, filesize=11.7 K 2024-11-10T09:27:50,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/0f948ce8e2484cf3a801ef34366adcc4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/0f948ce8e2484cf3a801ef34366adcc4 2024-11-10T09:27:50,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/0f948ce8e2484cf3a801ef34366adcc4, entries=150, sequenceid=55, filesize=11.7 K 2024-11-10T09:27:50,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for dedf9887fe3cb3c93b7ecf99bd810cef in 1251ms, sequenceid=55, compaction requested=true 2024-11-10T09:27:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:50,939 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:50,939 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:50,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/A is initiating minor compaction (all files) 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/B is initiating minor compaction (all files) 2024-11-10T09:27:50,940 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/A in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,940 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/B in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,940 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/961a7a287c9f47fe82c5b7c96d189f4b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/3f15da88c5094620a31c75af898a45ef, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/2d9c72beab60417980a5c43245c8c645] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=35.2 K 2024-11-10T09:27:50,940 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/e042482d642d4171bfd135fc17e07d27, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/908985dbfa814569894cf61656df5ee6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/634d62a6ebdb4bbb9d1510c0f474d188] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=99.1 K 2024-11-10T09:27:50,940 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/e042482d642d4171bfd135fc17e07d27, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/908985dbfa814569894cf61656df5ee6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/634d62a6ebdb4bbb9d1510c0f474d188] 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 961a7a287c9f47fe82c5b7c96d189f4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731230866348 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting e042482d642d4171bfd135fc17e07d27, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731230866348 2024-11-10T09:27:50,940 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f15da88c5094620a31c75af898a45ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230866382 2024-11-10T09:27:50,941 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d9c72beab60417980a5c43245c8c645, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731230868545 2024-11-10T09:27:50,941 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908985dbfa814569894cf61656df5ee6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230866382 2024-11-10T09:27:50,941 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 634d62a6ebdb4bbb9d1510c0f474d188, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731230868545 2024-11-10T09:27:50,947 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:50,948 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/27cc3749728141a1aa68e0c59f3de8ac is 50, key is test_row_0/B:col10/1731230868548/Put/seqid=0 2024-11-10T09:27:50,949 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:50,957 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111036179203c3ec431785ff62ae2a7267d5_dedf9887fe3cb3c93b7ecf99bd810cef store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742325_1501 (size=12104) 2024-11-10T09:27:50,958 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111036179203c3ec431785ff62ae2a7267d5_dedf9887fe3cb3c93b7ecf99bd810cef, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:50,959 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111036179203c3ec431785ff62ae2a7267d5_dedf9887fe3cb3c93b7ecf99bd810cef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:50,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742326_1502 (size=4469) 2024-11-10T09:27:50,962 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#A#compaction#429 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:50,963 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/f88e45a00ca44f5eb1304600e087f287 is 175, key is test_row_0/A:col10/1731230868548/Put/seqid=0 2024-11-10T09:27:50,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742327_1503 (size=31058) 2024-11-10T09:27:50,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T09:27:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:50,966 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-10T09:27:50,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a00bd2473d7d47afbe51f8562c5677a5_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230869712/Put/seqid=0 2024-11-10T09:27:50,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742328_1504 (size=12154) 2024-11-10T09:27:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:27:51,361 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/27cc3749728141a1aa68e0c59f3de8ac as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/27cc3749728141a1aa68e0c59f3de8ac 2024-11-10T09:27:51,365 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/B of dedf9887fe3cb3c93b7ecf99bd810cef into 27cc3749728141a1aa68e0c59f3de8ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:51,365 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:51,365 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/B, priority=13, startTime=1731230870939; duration=0sec 2024-11-10T09:27:51,365 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:51,365 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:B 2024-11-10T09:27:51,365 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:51,366 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:51,366 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/C is initiating minor compaction (all files) 2024-11-10T09:27:51,366 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/C in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:51,366 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d2b77755c8be47599fb473a408881afc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/247a3efb89ee42119d19ebf048dc0fae, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/0f948ce8e2484cf3a801ef34366adcc4] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=35.2 K 2024-11-10T09:27:51,367 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b77755c8be47599fb473a408881afc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731230866348 2024-11-10T09:27:51,367 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 247a3efb89ee42119d19ebf048dc0fae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731230866382 2024-11-10T09:27:51,367 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f948ce8e2484cf3a801ef34366adcc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731230868545 2024-11-10T09:27:51,369 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/f88e45a00ca44f5eb1304600e087f287 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/f88e45a00ca44f5eb1304600e087f287 2024-11-10T09:27:51,374 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/A of dedf9887fe3cb3c93b7ecf99bd810cef into f88e45a00ca44f5eb1304600e087f287(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:51,374 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:51,374 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/A, priority=13, startTime=1731230870939; duration=0sec 2024-11-10T09:27:51,374 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:51,374 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:A 2024-11-10T09:27:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,377 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a00bd2473d7d47afbe51f8562c5677a5_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a00bd2473d7d47afbe51f8562c5677a5_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/666341ecca2a415c8327785dcda73f16, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/666341ecca2a415c8327785dcda73f16 is 175, key is test_row_0/A:col10/1731230869712/Put/seqid=0 2024-11-10T09:27:51,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742329_1505 (size=30955) 2024-11-10T09:27:51,382 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/666341ecca2a415c8327785dcda73f16 2024-11-10T09:27:51,382 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:51,383 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/2a696426664748d989f16606168e467b is 50, key is test_row_0/C:col10/1731230868548/Put/seqid=0 2024-11-10T09:27:51,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742330_1506 (size=12104) 2024-11-10T09:27:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/53639f65c0f5429daa8446f4b41fa9e4 is 50, key is test_row_0/B:col10/1731230869712/Put/seqid=0 2024-11-10T09:27:51,393 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/2a696426664748d989f16606168e467b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/2a696426664748d989f16606168e467b 2024-11-10T09:27:51,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742331_1507 (size=12001) 2024-11-10T09:27:51,395 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/53639f65c0f5429daa8446f4b41fa9e4 2024-11-10T09:27:51,398 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/C of dedf9887fe3cb3c93b7ecf99bd810cef into 2a696426664748d989f16606168e467b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:51,398 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:51,399 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/C, priority=13, startTime=1731230870939; duration=0sec 2024-11-10T09:27:51,399 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:51,399 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:C 2024-11-10T09:27:51,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/d101ce5dfa36430c9766c7abbe59f126 is 50, key is test_row_0/C:col10/1731230869712/Put/seqid=0 2024-11-10T09:27:51,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742332_1508 (size=12001) 2024-11-10T09:27:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:27:51,826 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/d101ce5dfa36430c9766c7abbe59f126 2024-11-10T09:27:51,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/666341ecca2a415c8327785dcda73f16 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/666341ecca2a415c8327785dcda73f16 2024-11-10T09:27:51,832 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/666341ecca2a415c8327785dcda73f16, entries=150, sequenceid=76, filesize=30.2 K 2024-11-10T09:27:51,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/53639f65c0f5429daa8446f4b41fa9e4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/53639f65c0f5429daa8446f4b41fa9e4 2024-11-10T09:27:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,835 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/53639f65c0f5429daa8446f4b41fa9e4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T09:27:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/d101ce5dfa36430c9766c7abbe59f126 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d101ce5dfa36430c9766c7abbe59f126 2024-11-10T09:27:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d101ce5dfa36430c9766c7abbe59f126, entries=150, sequenceid=76, filesize=11.7 K 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,840 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for dedf9887fe3cb3c93b7ecf99bd810cef in 874ms, sequenceid=76, compaction requested=false 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:51,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-10T09:27:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T09:27:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3330 sec 2024-11-10T09:27:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.3370 sec 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-10T09:27:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109f73cbd435a048fc9e52cf6777dfe262_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230871894/Put/seqid=0 2024-11-10T09:27:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742334_1510 (size=24358) 2024-11-10T09:27:51,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230931928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:51,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230931929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:51,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230931930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:51,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230931930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230932036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230932045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230932045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230932045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230932246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230932255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230932255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230932255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,322 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:52,325 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109f73cbd435a048fc9e52cf6777dfe262_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109f73cbd435a048fc9e52cf6777dfe262_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:52,325 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4336c7226be34f839573bf55990b3f2d, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:52,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4336c7226be34f839573bf55990b3f2d is 175, key is test_row_0/A:col10/1731230871894/Put/seqid=0 2024-11-10T09:27:52,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742333_1509 (size=73991) 2024-11-10T09:27:52,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230932555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230932566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230932566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230932566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:52,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T09:27:52,656 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:52,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-10T09:27:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:27:52,658 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:52,658 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:52,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:52,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4336c7226be34f839573bf55990b3f2d 2024-11-10T09:27:52,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/96a60c8c73304b599b0454f1357bc377 is 50, key is test_row_0/B:col10/1731230871894/Put/seqid=0 2024-11-10T09:27:52,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742335_1511 (size=9657) 2024-11-10T09:27:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:27:52,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T09:27:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:52,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:52,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:52,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T09:27:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:52,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:52,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:27:53,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:53,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230933066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:53,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230933075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230933075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230933075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:53,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T09:27:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:53,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:53,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/96a60c8c73304b599b0454f1357bc377 2024-11-10T09:27:53,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/38bb381c0dd54e3db70d3ca59e8f915b is 50, key is test_row_0/C:col10/1731230871894/Put/seqid=0 2024-11-10T09:27:53,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742336_1512 (size=9657) 2024-11-10T09:27:53,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/38bb381c0dd54e3db70d3ca59e8f915b 2024-11-10T09:27:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4336c7226be34f839573bf55990b3f2d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4336c7226be34f839573bf55990b3f2d 2024-11-10T09:27:53,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4336c7226be34f839573bf55990b3f2d, entries=400, sequenceid=90, filesize=72.3 K 2024-11-10T09:27:53,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/96a60c8c73304b599b0454f1357bc377 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/96a60c8c73304b599b0454f1357bc377 2024-11-10T09:27:53,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/96a60c8c73304b599b0454f1357bc377, entries=100, sequenceid=90, filesize=9.4 K 2024-11-10T09:27:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/38bb381c0dd54e3db70d3ca59e8f915b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/38bb381c0dd54e3db70d3ca59e8f915b 2024-11-10T09:27:53,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/38bb381c0dd54e3db70d3ca59e8f915b, entries=100, sequenceid=90, filesize=9.4 K 2024-11-10T09:27:53,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dedf9887fe3cb3c93b7ecf99bd810cef in 1280ms, sequenceid=90, compaction requested=true 2024-11-10T09:27:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:53,175 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:53,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:53,176 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:53,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:53,177 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:53,177 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/B is initiating minor compaction (all files) 2024-11-10T09:27:53,177 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/B in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:53,177 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/27cc3749728141a1aa68e0c59f3de8ac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/53639f65c0f5429daa8446f4b41fa9e4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/96a60c8c73304b599b0454f1357bc377] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=33.0 K 2024-11-10T09:27:53,177 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136004 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:53,177 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/A is initiating minor compaction (all files) 2024-11-10T09:27:53,177 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/A in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:53,177 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/f88e45a00ca44f5eb1304600e087f287, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/666341ecca2a415c8327785dcda73f16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4336c7226be34f839573bf55990b3f2d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=132.8 K 2024-11-10T09:27:53,177 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:53,177 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/f88e45a00ca44f5eb1304600e087f287, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/666341ecca2a415c8327785dcda73f16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4336c7226be34f839573bf55990b3f2d] 2024-11-10T09:27:53,178 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 27cc3749728141a1aa68e0c59f3de8ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731230868545 2024-11-10T09:27:53,178 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88e45a00ca44f5eb1304600e087f287, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731230868545 2024-11-10T09:27:53,178 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 53639f65c0f5429daa8446f4b41fa9e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230869710 2024-11-10T09:27:53,178 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a60c8c73304b599b0454f1357bc377, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230871888 2024-11-10T09:27:53,178 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 666341ecca2a415c8327785dcda73f16, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230869710 2024-11-10T09:27:53,179 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4336c7226be34f839573bf55990b3f2d, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230871858 2024-11-10T09:27:53,184 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:53,184 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/a7606afbbda64fc78097d4a7a2616bb2 is 50, key is test_row_0/B:col10/1731230871894/Put/seqid=0 2024-11-10T09:27:53,192 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:53,197 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100dd0bcb91eff4f3eaadea4863dde8e06_dedf9887fe3cb3c93b7ecf99bd810cef store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:53,200 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100dd0bcb91eff4f3eaadea4863dde8e06_dedf9887fe3cb3c93b7ecf99bd810cef, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:53,200 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100dd0bcb91eff4f3eaadea4863dde8e06_dedf9887fe3cb3c93b7ecf99bd810cef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:53,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742337_1513 (size=12207) 2024-11-10T09:27:53,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742338_1514 (size=4469) 2024-11-10T09:27:53,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T09:27:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:53,269 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-10T09:27:53,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c331a6cc754e44479e22ecbe67f65905_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230871917/Put/seqid=0 2024-11-10T09:27:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:27:53,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742339_1515 (size=12154) 2024-11-10T09:27:53,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:53,294 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c331a6cc754e44479e22ecbe67f65905_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c331a6cc754e44479e22ecbe67f65905_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:53,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4fcf2c3f89a64cf48c9b4f6a01f78af7, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:53,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4fcf2c3f89a64cf48c9b4f6a01f78af7 is 175, key is test_row_0/A:col10/1731230871917/Put/seqid=0 2024-11-10T09:27:53,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742340_1516 (size=30955) 2024-11-10T09:27:53,649 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#A#compaction#438 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:53,649 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/18ca409e386e429a9a118c87353f1057 is 175, key is test_row_0/A:col10/1731230871894/Put/seqid=0 2024-11-10T09:27:53,677 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/a7606afbbda64fc78097d4a7a2616bb2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/a7606afbbda64fc78097d4a7a2616bb2 2024-11-10T09:27:53,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742341_1517 (size=31268) 2024-11-10T09:27:53,682 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/B of dedf9887fe3cb3c93b7ecf99bd810cef into a7606afbbda64fc78097d4a7a2616bb2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:53,682 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:53,682 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/B, priority=13, startTime=1731230873175; duration=0sec 2024-11-10T09:27:53,682 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/18ca409e386e429a9a118c87353f1057 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/18ca409e386e429a9a118c87353f1057 2024-11-10T09:27:53,682 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:53,682 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:B 2024-11-10T09:27:53,682 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:53,683 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:53,683 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/C is initiating minor compaction (all files) 2024-11-10T09:27:53,683 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/C in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:53,683 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/2a696426664748d989f16606168e467b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d101ce5dfa36430c9766c7abbe59f126, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/38bb381c0dd54e3db70d3ca59e8f915b] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=33.0 K 2024-11-10T09:27:53,684 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a696426664748d989f16606168e467b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731230868545 2024-11-10T09:27:53,684 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting d101ce5dfa36430c9766c7abbe59f126, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731230869710 2024-11-10T09:27:53,685 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 38bb381c0dd54e3db70d3ca59e8f915b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230871888 2024-11-10T09:27:53,687 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/A of dedf9887fe3cb3c93b7ecf99bd810cef into 18ca409e386e429a9a118c87353f1057(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:53,687 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:53,687 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/A, priority=13, startTime=1731230873175; duration=0sec 2024-11-10T09:27:53,687 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:53,687 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:A 2024-11-10T09:27:53,691 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:53,691 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/a0c37cb5450d41b5aafca8ce1096ece2 is 50, key is test_row_0/C:col10/1731230871894/Put/seqid=0 2024-11-10T09:27:53,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742342_1518 (size=12207) 2024-11-10T09:27:53,702 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4fcf2c3f89a64cf48c9b4f6a01f78af7 2024-11-10T09:27:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/01321d2a3f294ae1bdb9ea37ba9a1709 is 50, key is test_row_0/B:col10/1731230871917/Put/seqid=0 2024-11-10T09:27:53,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742343_1519 (size=12001) 2024-11-10T09:27:53,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T09:27:53,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T09:27:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:27:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:54,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230934091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230934093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,103 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/a0c37cb5450d41b5aafca8ce1096ece2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a0c37cb5450d41b5aafca8ce1096ece2 2024-11-10T09:27:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230934100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230934100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,103 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:54,104 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_2' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,107 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/C of dedf9887fe3cb3c93b7ecf99bd810cef into a0c37cb5450d41b5aafca8ce1096ece2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:54,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:54,107 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/C, priority=13, startTime=1731230873176; duration=0sec 2024-11-10T09:27:54,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:54,107 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:C 2024-11-10T09:27:54,115 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/01321d2a3f294ae1bdb9ea37ba9a1709 2024-11-10T09:27:54,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/1aa1caa057bb460786d061751af56980 is 50, key is test_row_0/C:col10/1731230871917/Put/seqid=0 2024-11-10T09:27:54,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742344_1520 (size=12001) 2024-11-10T09:27:54,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230934195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230934206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230934405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230934415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,524 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/1aa1caa057bb460786d061751af56980 2024-11-10T09:27:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/4fcf2c3f89a64cf48c9b4f6a01f78af7 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4fcf2c3f89a64cf48c9b4f6a01f78af7 2024-11-10T09:27:54,530 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4fcf2c3f89a64cf48c9b4f6a01f78af7, entries=150, sequenceid=115, filesize=30.2 K 2024-11-10T09:27:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/01321d2a3f294ae1bdb9ea37ba9a1709 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/01321d2a3f294ae1bdb9ea37ba9a1709 2024-11-10T09:27:54,534 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/01321d2a3f294ae1bdb9ea37ba9a1709, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T09:27:54,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/1aa1caa057bb460786d061751af56980 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/1aa1caa057bb460786d061751af56980 2024-11-10T09:27:54,537 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/1aa1caa057bb460786d061751af56980, entries=150, sequenceid=115, filesize=11.7 K 2024-11-10T09:27:54,538 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dedf9887fe3cb3c93b7ecf99bd810cef in 1269ms, sequenceid=115, compaction requested=false 2024-11-10T09:27:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:54,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-10T09:27:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-10T09:27:54,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T09:27:54,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-10T09:27:54,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8850 sec 2024-11-10T09:27:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:54,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:54,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-10T09:27:54,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a0c3d9c3fbe14b13853e47440d0e6dc5_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230874089/Put/seqid=0 2024-11-10T09:27:54,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742345_1521 (size=14644) 2024-11-10T09:27:54,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230934670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230934716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230934725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230934775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T09:27:54,806 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:54,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-10T09:27:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:27:54,808 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:54,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:54,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:27:54,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:27:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:54,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:54,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230934985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:55,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:55,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a0c3d9c3fbe14b13853e47440d0e6dc5_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a0c3d9c3fbe14b13853e47440d0e6dc5_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:55,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0b0ff027453f486583a2709546b3b2d1, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:55,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0b0ff027453f486583a2709546b3b2d1 is 175, key is test_row_0/A:col10/1731230874089/Put/seqid=0 2024-11-10T09:27:55,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742346_1522 (size=39599) 2024-11-10T09:27:55,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:27:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:27:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230935225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230935236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:55,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:27:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230935295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:55,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0b0ff027453f486583a2709546b3b2d1 2024-11-10T09:27:55,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:27:55,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:55,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/4d9b933a332d41aebcf8882efd26d004 is 50, key is test_row_0/B:col10/1731230874089/Put/seqid=0 2024-11-10T09:27:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:27:55,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742347_1523 (size=12051) 2024-11-10T09:27:55,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:27:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:27:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230935805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:55,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:55,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/4d9b933a332d41aebcf8882efd26d004 2024-11-10T09:27:55,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/4ca58bfc0bfa4268ae22cd6ed1a9123a is 50, key is test_row_0/C:col10/1731230874089/Put/seqid=0 2024-11-10T09:27:55,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742348_1524 (size=12051) 2024-11-10T09:27:55,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/4ca58bfc0bfa4268ae22cd6ed1a9123a 2024-11-10T09:27:55,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0b0ff027453f486583a2709546b3b2d1 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0b0ff027453f486583a2709546b3b2d1 2024-11-10T09:27:55,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0b0ff027453f486583a2709546b3b2d1, entries=200, sequenceid=130, filesize=38.7 K 2024-11-10T09:27:55,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/4d9b933a332d41aebcf8882efd26d004 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/4d9b933a332d41aebcf8882efd26d004 2024-11-10T09:27:55,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/4d9b933a332d41aebcf8882efd26d004, entries=150, sequenceid=130, filesize=11.8 K 2024-11-10T09:27:55,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/4ca58bfc0bfa4268ae22cd6ed1a9123a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4ca58bfc0bfa4268ae22cd6ed1a9123a 2024-11-10T09:27:55,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4ca58bfc0bfa4268ae22cd6ed1a9123a, entries=150, sequenceid=130, filesize=11.8 K 2024-11-10T09:27:55,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dedf9887fe3cb3c93b7ecf99bd810cef in 1268ms, sequenceid=130, compaction requested=true 2024-11-10T09:27:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:55,866 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:55,866 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:55,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:55,867 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101822 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:55,867 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:55,867 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/B is initiating minor compaction (all files) 2024-11-10T09:27:55,867 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/A is initiating minor compaction (all files) 2024-11-10T09:27:55,867 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/A in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,867 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/B in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,867 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/a7606afbbda64fc78097d4a7a2616bb2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/01321d2a3f294ae1bdb9ea37ba9a1709, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/4d9b933a332d41aebcf8882efd26d004] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=35.4 K 2024-11-10T09:27:55,867 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/18ca409e386e429a9a118c87353f1057, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4fcf2c3f89a64cf48c9b4f6a01f78af7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0b0ff027453f486583a2709546b3b2d1] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=99.4 K 2024-11-10T09:27:55,867 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,867 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/18ca409e386e429a9a118c87353f1057, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4fcf2c3f89a64cf48c9b4f6a01f78af7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0b0ff027453f486583a2709546b3b2d1] 2024-11-10T09:27:55,868 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a7606afbbda64fc78097d4a7a2616bb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230869710 2024-11-10T09:27:55,868 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ca409e386e429a9a118c87353f1057, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230869710 2024-11-10T09:27:55,869 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fcf2c3f89a64cf48c9b4f6a01f78af7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230871917 2024-11-10T09:27:55,869 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 01321d2a3f294ae1bdb9ea37ba9a1709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230871917 2024-11-10T09:27:55,869 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b0ff027453f486583a2709546b3b2d1, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731230874089 2024-11-10T09:27:55,869 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d9b933a332d41aebcf8882efd26d004, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731230874089 2024-11-10T09:27:55,874 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:55,875 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:55,876 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/7058cc72939749c5834bc22a9f247bac is 50, key is test_row_0/B:col10/1731230874089/Put/seqid=0 2024-11-10T09:27:55,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T09:27:55,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,876 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-10T09:27:55,878 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411105b1960f13c824148972cb98c3b258c3b_dedf9887fe3cb3c93b7ecf99bd810cef store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:55,880 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411105b1960f13c824148972cb98c3b258c3b_dedf9887fe3cb3c93b7ecf99bd810cef, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:55,880 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105b1960f13c824148972cb98c3b258c3b_dedf9887fe3cb3c93b7ecf99bd810cef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:55,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742349_1525 (size=12359) 2024-11-10T09:27:55,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742350_1526 (size=4469) 2024-11-10T09:27:55,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103e3543ecb52d4235b22f71defe2b19a7_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230874665/Put/seqid=0 2024-11-10T09:27:55,893 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#A#compaction#446 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:55,893 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/69ed34a7ddb5406ea244987491160c48 is 175, key is test_row_0/A:col10/1731230874089/Put/seqid=0 2024-11-10T09:27:55,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742351_1527 (size=12304) 2024-11-10T09:27:55,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742352_1528 (size=31313) 2024-11-10T09:27:55,900 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/69ed34a7ddb5406ea244987491160c48 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/69ed34a7ddb5406ea244987491160c48 2024-11-10T09:27:55,905 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/A of dedf9887fe3cb3c93b7ecf99bd810cef into 69ed34a7ddb5406ea244987491160c48(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:55,905 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:55,905 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/A, priority=13, startTime=1731230875866; duration=0sec 2024-11-10T09:27:55,905 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:55,905 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:A 2024-11-10T09:27:55,905 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:55,906 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:55,906 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/C is initiating minor compaction (all files) 2024-11-10T09:27:55,906 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/C in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:55,906 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a0c37cb5450d41b5aafca8ce1096ece2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/1aa1caa057bb460786d061751af56980, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4ca58bfc0bfa4268ae22cd6ed1a9123a] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=35.4 K 2024-11-10T09:27:55,907 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c37cb5450d41b5aafca8ce1096ece2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731230869710 2024-11-10T09:27:55,907 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa1caa057bb460786d061751af56980, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731230871917 2024-11-10T09:27:55,908 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ca58bfc0bfa4268ae22cd6ed1a9123a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731230874089 2024-11-10T09:27:55,912 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:27:55,913 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/8e70992c2df2470f90b5dfa3b01b0cc9 is 50, key is test_row_0/C:col10/1731230874089/Put/seqid=0 2024-11-10T09:27:55,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742353_1529 (size=12359) 2024-11-10T09:27:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:27:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:56,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230936133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230936136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230936235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,238 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230936245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,246 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230936246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230936246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,290 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/7058cc72939749c5834bc22a9f247bac as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/7058cc72939749c5834bc22a9f247bac 2024-11-10T09:27:56,293 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/B of dedf9887fe3cb3c93b7ecf99bd810cef into 7058cc72939749c5834bc22a9f247bac(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:56,293 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:56,293 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/B, priority=13, startTime=1731230875866; duration=0sec 2024-11-10T09:27:56,293 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:56,294 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:B 2024-11-10T09:27:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:56,298 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103e3543ecb52d4235b22f71defe2b19a7_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e3543ecb52d4235b22f71defe2b19a7_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/a9fd586dc161490d9e11d60dca7ba38b, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/a9fd586dc161490d9e11d60dca7ba38b is 175, key is test_row_0/A:col10/1731230874665/Put/seqid=0 2024-11-10T09:27:56,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742354_1530 (size=31105) 2024-11-10T09:27:56,319 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/8e70992c2df2470f90b5dfa3b01b0cc9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/8e70992c2df2470f90b5dfa3b01b0cc9 2024-11-10T09:27:56,324 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/C of dedf9887fe3cb3c93b7ecf99bd810cef into 8e70992c2df2470f90b5dfa3b01b0cc9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:27:56,324 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:56,324 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/C, priority=13, startTime=1731230875866; duration=0sec 2024-11-10T09:27:56,324 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:56,324 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:C 2024-11-10T09:27:56,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230936455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230936455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,702 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/a9fd586dc161490d9e11d60dca7ba38b 2024-11-10T09:27:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/30523d4cfa8e4245957ec785485e4e3a is 50, key is test_row_0/B:col10/1731230874665/Put/seqid=0 2024-11-10T09:27:56,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742355_1531 (size=12151) 2024-11-10T09:27:56,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230936766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230936766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230936815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:56,818 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:27:57,112 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/30523d4cfa8e4245957ec785485e4e3a 2024-11-10T09:27:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/41b137e2c7d84c00a903370a81ea629c is 50, key is test_row_0/C:col10/1731230874665/Put/seqid=0 2024-11-10T09:27:57,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742356_1532 (size=12151) 2024-11-10T09:27:57,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230937276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:57,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230937276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:57,521 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/41b137e2c7d84c00a903370a81ea629c 2024-11-10T09:27:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/a9fd586dc161490d9e11d60dca7ba38b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/a9fd586dc161490d9e11d60dca7ba38b 2024-11-10T09:27:57,527 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/a9fd586dc161490d9e11d60dca7ba38b, entries=150, sequenceid=154, filesize=30.4 K 2024-11-10T09:27:57,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/30523d4cfa8e4245957ec785485e4e3a as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/30523d4cfa8e4245957ec785485e4e3a 2024-11-10T09:27:57,530 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/30523d4cfa8e4245957ec785485e4e3a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T09:27:57,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/41b137e2c7d84c00a903370a81ea629c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/41b137e2c7d84c00a903370a81ea629c 2024-11-10T09:27:57,533 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/41b137e2c7d84c00a903370a81ea629c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-10T09:27:57,534 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dedf9887fe3cb3c93b7ecf99bd810cef in 1658ms, sequenceid=154, compaction requested=false 2024-11-10T09:27:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-10T09:27:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-10T09:27:57,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-10T09:27:57,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7260 sec 2024-11-10T09:27:57,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7300 sec 2024-11-10T09:27:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:58,257 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:27:58,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-10T09:27:58,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110672ce65acbcc46479fd7f5753aca2ef6_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230876132/Put/seqid=0 2024-11-10T09:27:58,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742357_1533 (size=12304) 2024-11-10T09:27:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230938291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230938290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230938294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230938294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230938406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230938406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230938406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230938406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230938616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230938616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230938616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230938616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,666 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:27:58,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110672ce65acbcc46479fd7f5753aca2ef6_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110672ce65acbcc46479fd7f5753aca2ef6_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:27:58,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/26579e00f80c43f4871ce120a6e98b7d, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:58,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/26579e00f80c43f4871ce120a6e98b7d is 175, key is test_row_0/A:col10/1731230876132/Put/seqid=0 2024-11-10T09:27:58,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742358_1534 (size=31105) 2024-11-10T09:27:58,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230938826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,828 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:27:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230938926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230938926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230938926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230938926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:58,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T09:27:58,976 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:27:58,976 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:27:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-10T09:27:58,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:27:58,978 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:27:58,978 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:27:58,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:27:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:27:59,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:27:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:59,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/26579e00f80c43f4871ce120a6e98b7d 2024-11-10T09:27:59,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/9c32aba15f224dda85c66710b9baf1f2 is 50, key is test_row_0/B:col10/1731230876132/Put/seqid=0 2024-11-10T09:27:59,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742359_1535 (size=12151) 2024-11-10T09:27:59,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:27:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:27:59,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:27:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230939436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230939436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230939436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230939436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:27:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:27:59,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/9c32aba15f224dda85c66710b9baf1f2 2024-11-10T09:27:59,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/a5f8949a416e4cee823ead8604151400 is 50, key is test_row_0/C:col10/1731230876132/Put/seqid=0 2024-11-10T09:27:59,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742360_1536 (size=12151) 2024-11-10T09:27:59,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:27:59,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:59,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:27:59,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:27:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:27:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:27:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:27:59,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/a5f8949a416e4cee823ead8604151400 2024-11-10T09:27:59,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/26579e00f80c43f4871ce120a6e98b7d as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/26579e00f80c43f4871ce120a6e98b7d 2024-11-10T09:27:59,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/26579e00f80c43f4871ce120a6e98b7d, entries=150, sequenceid=170, filesize=30.4 K 2024-11-10T09:27:59,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/9c32aba15f224dda85c66710b9baf1f2 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/9c32aba15f224dda85c66710b9baf1f2 2024-11-10T09:27:59,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/9c32aba15f224dda85c66710b9baf1f2, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T09:27:59,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/a5f8949a416e4cee823ead8604151400 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a5f8949a416e4cee823ead8604151400 2024-11-10T09:27:59,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a5f8949a416e4cee823ead8604151400, entries=150, sequenceid=170, filesize=11.9 K 2024-11-10T09:27:59,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for dedf9887fe3cb3c93b7ecf99bd810cef in 1726ms, sequenceid=170, compaction requested=true 2024-11-10T09:27:59,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:27:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:27:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:59,984 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:27:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:27:59,984 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:27:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:27:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:27:59,984 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93523 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:59,984 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:27:59,984 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/B is initiating minor compaction (all files) 2024-11-10T09:27:59,984 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/A is initiating minor compaction (all files) 2024-11-10T09:27:59,985 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/B in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,985 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/A in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,985 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/69ed34a7ddb5406ea244987491160c48, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/a9fd586dc161490d9e11d60dca7ba38b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/26579e00f80c43f4871ce120a6e98b7d] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=91.3 K 2024-11-10T09:27:59,985 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/7058cc72939749c5834bc22a9f247bac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/30523d4cfa8e4245957ec785485e4e3a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/9c32aba15f224dda85c66710b9baf1f2] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=35.8 K 2024-11-10T09:27:59,985 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:27:59,985 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/69ed34a7ddb5406ea244987491160c48, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/a9fd586dc161490d9e11d60dca7ba38b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/26579e00f80c43f4871ce120a6e98b7d] 2024-11-10T09:27:59,986 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 7058cc72939749c5834bc22a9f247bac, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731230874089 2024-11-10T09:27:59,986 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69ed34a7ddb5406ea244987491160c48, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731230874089 2024-11-10T09:27:59,986 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 30523d4cfa8e4245957ec785485e4e3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230874645 2024-11-10T09:27:59,986 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9fd586dc161490d9e11d60dca7ba38b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230874645 2024-11-10T09:27:59,988 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26579e00f80c43f4871ce120a6e98b7d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230876132 2024-11-10T09:27:59,988 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c32aba15f224dda85c66710b9baf1f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230876132 2024-11-10T09:27:59,993 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:27:59,999 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110a2ff5a1e4f7543d48c4f30ca654405fc_dedf9887fe3cb3c93b7ecf99bd810cef store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:00,000 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:00,001 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/47dde69a66b4460eb03b06a511664c44 is 50, key is test_row_0/B:col10/1731230876132/Put/seqid=0 2024-11-10T09:28:00,001 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110a2ff5a1e4f7543d48c4f30ca654405fc_dedf9887fe3cb3c93b7ecf99bd810cef, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:00,001 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a2ff5a1e4f7543d48c4f30ca654405fc_dedf9887fe3cb3c93b7ecf99bd810cef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:00,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742362_1538 (size=4469) 2024-11-10T09:28:00,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742361_1537 (size=12561) 2024-11-10T09:28:00,010 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/47dde69a66b4460eb03b06a511664c44 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/47dde69a66b4460eb03b06a511664c44 2024-11-10T09:28:00,014 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/B of dedf9887fe3cb3c93b7ecf99bd810cef into 47dde69a66b4460eb03b06a511664c44(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:00,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:00,014 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/B, priority=13, startTime=1731230879984; duration=0sec 2024-11-10T09:28:00,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:28:00,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:B 2024-11-10T09:28:00,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:28:00,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:28:00,014 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/C is initiating minor compaction (all files) 2024-11-10T09:28:00,014 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/C in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:00,015 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/8e70992c2df2470f90b5dfa3b01b0cc9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/41b137e2c7d84c00a903370a81ea629c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a5f8949a416e4cee823ead8604151400] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=35.8 K 2024-11-10T09:28:00,015 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e70992c2df2470f90b5dfa3b01b0cc9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731230874089 2024-11-10T09:28:00,015 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b137e2c7d84c00a903370a81ea629c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731230874645 2024-11-10T09:28:00,015 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f8949a416e4cee823ead8604151400, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230876132 2024-11-10T09:28:00,020 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:00,021 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/b910a105f6a045beb4c6c36387976d41 is 50, key is test_row_0/C:col10/1731230876132/Put/seqid=0 2024-11-10T09:28:00,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742363_1539 (size=12561) 2024-11-10T09:28:00,026 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/b910a105f6a045beb4c6c36387976d41 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/b910a105f6a045beb4c6c36387976d41 2024-11-10T09:28:00,029 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/C of dedf9887fe3cb3c93b7ecf99bd810cef into b910a105f6a045beb4c6c36387976d41(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:00,029 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:00,029 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/C, priority=13, startTime=1731230879984; duration=0sec 2024-11-10T09:28:00,029 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:00,030 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:C 2024-11-10T09:28:00,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T09:28:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:00,045 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:28:00,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bc07fd7d7b154ae3b423956a6dadd11a_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230878290/Put/seqid=0 2024-11-10T09:28:00,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742364_1540 (size=12304) 2024-11-10T09:28:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:28:00,405 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#A#compaction#455 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:00,406 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/7a625b980ccf40aca46095ae4e7745fe is 175, key is test_row_0/A:col10/1731230876132/Put/seqid=0 2024-11-10T09:28:00,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742365_1541 (size=31515) 2024-11-10T09:28:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:28:00,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:28:00,456 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bc07fd7d7b154ae3b423956a6dadd11a_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bc07fd7d7b154ae3b423956a6dadd11a_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:00,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/333467eaf5a14355ae5cd15e9accfe4c, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:00,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/333467eaf5a14355ae5cd15e9accfe4c is 175, key is test_row_0/A:col10/1731230878290/Put/seqid=0 2024-11-10T09:28:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230940457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230940458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230940460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742366_1542 (size=31105) 2024-11-10T09:28:00,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230940461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230940565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230940566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230940566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230940566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230940775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230940775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230940775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230940776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:00,812 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/7a625b980ccf40aca46095ae4e7745fe as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7a625b980ccf40aca46095ae4e7745fe 2024-11-10T09:28:00,815 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/A of dedf9887fe3cb3c93b7ecf99bd810cef into 7a625b980ccf40aca46095ae4e7745fe(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:00,815 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:00,815 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/A, priority=13, startTime=1731230879983; duration=0sec 2024-11-10T09:28:00,815 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:00,815 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:A 2024-11-10T09:28:00,863 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/333467eaf5a14355ae5cd15e9accfe4c 2024-11-10T09:28:00,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/5b604bac8975408383359a19fd17f1f8 is 50, key is test_row_0/B:col10/1731230878290/Put/seqid=0 2024-11-10T09:28:00,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742367_1543 (size=12151) 2024-11-10T09:28:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230941086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230941086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230941086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230941086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:28:01,272 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/5b604bac8975408383359a19fd17f1f8 2024-11-10T09:28:01,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/df2c46071fee4c288622e4f0e49d6598 is 50, key is test_row_0/C:col10/1731230878290/Put/seqid=0 2024-11-10T09:28:01,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742368_1544 (size=12151) 2024-11-10T09:28:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230941596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230941596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230941596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230941596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:01,680 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/df2c46071fee4c288622e4f0e49d6598 2024-11-10T09:28:01,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/333467eaf5a14355ae5cd15e9accfe4c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/333467eaf5a14355ae5cd15e9accfe4c 2024-11-10T09:28:01,686 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/333467eaf5a14355ae5cd15e9accfe4c, entries=150, sequenceid=195, filesize=30.4 K 2024-11-10T09:28:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/5b604bac8975408383359a19fd17f1f8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5b604bac8975408383359a19fd17f1f8 2024-11-10T09:28:01,690 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5b604bac8975408383359a19fd17f1f8, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:28:01,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/df2c46071fee4c288622e4f0e49d6598 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/df2c46071fee4c288622e4f0e49d6598 2024-11-10T09:28:01,694 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/df2c46071fee4c288622e4f0e49d6598, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T09:28:01,695 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for dedf9887fe3cb3c93b7ecf99bd810cef in 1650ms, sequenceid=195, compaction requested=false 2024-11-10T09:28:01,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:01,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:01,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-10T09:28:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-10T09:28:01,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-10T09:28:01,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7180 sec 2024-11-10T09:28:01,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7220 sec 2024-11-10T09:28:02,146 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T09:28:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:02,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:28:02,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T09:28:02,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110626c560b199c45c694f286c1f38f24a9_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230880460/Put/seqid=0 2024-11-10T09:28:02,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742369_1545 (size=14794) 2024-11-10T09:28:02,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230942638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230942639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230942640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230942651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230942746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230942755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230942755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230942765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51594 deadline: 1731230942835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,836 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 802346a334f5,32883,1731230704436 for 'test_row_1' in dedf9887fe3cb3c93b7ecf99bd810cef of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T09:28:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230942955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230942965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230942965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:02,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230942975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:02,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,015 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:28:03,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110626c560b199c45c694f286c1f38f24a9_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110626c560b199c45c694f286c1f38f24a9_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:03,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/2b25721eba614ffd84ece9a573f5da9c, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:03,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/2b25721eba614ffd84ece9a573f5da9c is 175, key is test_row_0/A:col10/1731230880460/Put/seqid=0 2024-11-10T09:28:03,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742370_1546 (size=39749) 2024-11-10T09:28:03,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/2b25721eba614ffd84ece9a573f5da9c 2024-11-10T09:28:03,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/e11ec53a2e4241a28346465d2b7c0c26 is 50, key is test_row_0/B:col10/1731230880460/Put/seqid=0 2024-11-10T09:28:03,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742371_1547 (size=12151) 2024-11-10T09:28:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T09:28:03,136 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:28:03,136 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-10T09:28:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-10T09:28:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:28:03,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T09:28:03,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T09:28:03,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T09:28:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:28:03,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230943266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230943275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230943275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230943285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:28:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:28:03,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/e11ec53a2e4241a28346465d2b7c0c26 2024-11-10T09:28:03,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:28:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:28:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/cbd3fcfbbaf448c2a0ff4deab429cde5 is 50, key is test_row_0/C:col10/1731230880460/Put/seqid=0 2024-11-10T09:28:03,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742372_1548 (size=12151) 2024-11-10T09:28:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:28:03,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:28:03,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:28:03,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:28:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:28:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T09:28:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:28:03,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230943776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230943785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230943785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230943796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:03,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/cbd3fcfbbaf448c2a0ff4deab429cde5 2024-11-10T09:28:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/2b25721eba614ffd84ece9a573f5da9c as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/2b25721eba614ffd84ece9a573f5da9c 2024-11-10T09:28:03,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/2b25721eba614ffd84ece9a573f5da9c, entries=200, sequenceid=210, filesize=38.8 K 2024-11-10T09:28:03,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/e11ec53a2e4241a28346465d2b7c0c26 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/e11ec53a2e4241a28346465d2b7c0c26 2024-11-10T09:28:03,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/e11ec53a2e4241a28346465d2b7c0c26, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T09:28:03,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/cbd3fcfbbaf448c2a0ff4deab429cde5 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/cbd3fcfbbaf448c2a0ff4deab429cde5 2024-11-10T09:28:03,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/cbd3fcfbbaf448c2a0ff4deab429cde5, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T09:28:03,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for dedf9887fe3cb3c93b7ecf99bd810cef in 1257ms, sequenceid=210, compaction requested=true 2024-11-10T09:28:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:28:03,864 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:28:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:28:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:03,864 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:28:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:28:03,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/A is initiating minor compaction (all files) 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/B is initiating minor compaction (all files) 2024-11-10T09:28:03,867 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/B in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,867 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/A in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,867 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/47dde69a66b4460eb03b06a511664c44, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5b604bac8975408383359a19fd17f1f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/e11ec53a2e4241a28346465d2b7c0c26] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=36.0 K 2024-11-10T09:28:03,867 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7a625b980ccf40aca46095ae4e7745fe, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/333467eaf5a14355ae5cd15e9accfe4c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/2b25721eba614ffd84ece9a573f5da9c] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=100.0 K 2024-11-10T09:28:03,867 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7a625b980ccf40aca46095ae4e7745fe, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/333467eaf5a14355ae5cd15e9accfe4c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/2b25721eba614ffd84ece9a573f5da9c] 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 47dde69a66b4460eb03b06a511664c44, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230876132 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a625b980ccf40aca46095ae4e7745fe, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230876132 2024-11-10T09:28:03,867 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b604bac8975408383359a19fd17f1f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230878288 2024-11-10T09:28:03,868 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 333467eaf5a14355ae5cd15e9accfe4c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230878288 2024-11-10T09:28:03,868 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting e11ec53a2e4241a28346465d2b7c0c26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230880459 2024-11-10T09:28:03,868 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b25721eba614ffd84ece9a573f5da9c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230880449 2024-11-10T09:28:03,872 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:03,873 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#B#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:03,873 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/8a252673245c48c2b1a79b3b8121b4f8 is 50, key is test_row_0/B:col10/1731230880460/Put/seqid=0 2024-11-10T09:28:03,874 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ee83140c84bd4223bf1e40e70c81b178_dedf9887fe3cb3c93b7ecf99bd810cef store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:03,876 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ee83140c84bd4223bf1e40e70c81b178_dedf9887fe3cb3c93b7ecf99bd810cef, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:03,876 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ee83140c84bd4223bf1e40e70c81b178_dedf9887fe3cb3c93b7ecf99bd810cef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742373_1549 (size=12663) 2024-11-10T09:28:03,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742374_1550 (size=4469) 2024-11-10T09:28:03,894 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#A#compaction#465 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:03,894 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/231ac36339ed402b9e478cc08ad3a14b is 175, key is test_row_0/A:col10/1731230880460/Put/seqid=0 2024-11-10T09:28:03,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742375_1551 (size=31617) 2024-11-10T09:28:03,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T09:28:03,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,901 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/231ac36339ed402b9e478cc08ad3a14b as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/231ac36339ed402b9e478cc08ad3a14b 2024-11-10T09:28:03,901 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:28:03,905 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/A of dedf9887fe3cb3c93b7ecf99bd810cef into 231ac36339ed402b9e478cc08ad3a14b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:03,905 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:03,905 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/A, priority=13, startTime=1731230883864; duration=0sec 2024-11-10T09:28:03,906 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:28:03,906 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:A 2024-11-10T09:28:03,906 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:28:03,906 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:28:03,906 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/C is initiating minor compaction (all files) 2024-11-10T09:28:03,907 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/C in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:03,907 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/b910a105f6a045beb4c6c36387976d41, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/df2c46071fee4c288622e4f0e49d6598, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/cbd3fcfbbaf448c2a0ff4deab429cde5] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=36.0 K 2024-11-10T09:28:03,907 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting b910a105f6a045beb4c6c36387976d41, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731230876132 2024-11-10T09:28:03,907 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting df2c46071fee4c288622e4f0e49d6598, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731230878288 2024-11-10T09:28:03,908 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd3fcfbbaf448c2a0ff4deab429cde5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230880459 2024-11-10T09:28:03,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c948230ca95a4134b11ca0ae1b92f31c_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230882638/Put/seqid=0 2024-11-10T09:28:03,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742376_1552 (size=12304) 2024-11-10T09:28:03,914 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:03,914 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/43e6ca91bd5549c289ef1c0c483ce813 is 50, key is test_row_0/C:col10/1731230880460/Put/seqid=0 2024-11-10T09:28:03,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742377_1553 (size=12663) 2024-11-10T09:28:03,921 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/43e6ca91bd5549c289ef1c0c483ce813 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/43e6ca91bd5549c289ef1c0c483ce813 2024-11-10T09:28:03,925 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/C of dedf9887fe3cb3c93b7ecf99bd810cef into 43e6ca91bd5549c289ef1c0c483ce813(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:03,925 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:03,925 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/C, priority=13, startTime=1731230883864; duration=0sec 2024-11-10T09:28:03,925 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:03,925 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:C 2024-11-10T09:28:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:28:04,294 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/8a252673245c48c2b1a79b3b8121b4f8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/8a252673245c48c2b1a79b3b8121b4f8 2024-11-10T09:28:04,298 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/B of dedf9887fe3cb3c93b7ecf99bd810cef into 8a252673245c48c2b1a79b3b8121b4f8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:04,298 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:04,298 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/B, priority=13, startTime=1731230883864; duration=0sec 2024-11-10T09:28:04,298 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:04,298 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:B 2024-11-10T09:28:04,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:28:04,314 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c948230ca95a4134b11ca0ae1b92f31c_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c948230ca95a4134b11ca0ae1b92f31c_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:04,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0ceb269d9f924f4495b4e1853cf37f0f, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0ceb269d9f924f4495b4e1853cf37f0f is 175, key is test_row_0/A:col10/1731230882638/Put/seqid=0 2024-11-10T09:28:04,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742378_1554 (size=31105) 2024-11-10T09:28:04,720 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0ceb269d9f924f4495b4e1853cf37f0f 2024-11-10T09:28:04,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/5dc24e008df4423984afef3080b2ebab is 50, key is test_row_0/B:col10/1731230882638/Put/seqid=0 2024-11-10T09:28:04,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742379_1555 (size=12151) 2024-11-10T09:28:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:04,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. as already flushing 2024-11-10T09:28:04,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230944807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230944808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:04,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230944810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230944812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230944915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230944915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230944915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:04,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:04,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230944925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230945126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230945126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230945126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,128 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/5dc24e008df4423984afef3080b2ebab 2024-11-10T09:28:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/ccda31123231495e9b21f5cfb1c3a3fd is 50, key is test_row_0/C:col10/1731230882638/Put/seqid=0 2024-11-10T09:28:05,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742380_1556 (size=12151) 2024-11-10T09:28:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230945135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:28:05,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230945436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230945436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230945436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230945445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,537 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/ccda31123231495e9b21f5cfb1c3a3fd 2024-11-10T09:28:05,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/0ceb269d9f924f4495b4e1853cf37f0f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0ceb269d9f924f4495b4e1853cf37f0f 2024-11-10T09:28:05,543 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0ceb269d9f924f4495b4e1853cf37f0f, entries=150, sequenceid=233, filesize=30.4 K 2024-11-10T09:28:05,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/5dc24e008df4423984afef3080b2ebab as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5dc24e008df4423984afef3080b2ebab 2024-11-10T09:28:05,546 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5dc24e008df4423984afef3080b2ebab, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T09:28:05,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/ccda31123231495e9b21f5cfb1c3a3fd as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/ccda31123231495e9b21f5cfb1c3a3fd 2024-11-10T09:28:05,550 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/ccda31123231495e9b21f5cfb1c3a3fd, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T09:28:05,550 INFO [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for dedf9887fe3cb3c93b7ecf99bd810cef in 1649ms, sequenceid=233, compaction requested=false 2024-11-10T09:28:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:05,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/802346a334f5:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-10T09:28:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-10T09:28:05,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-10T09:28:05,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4140 sec 2024-11-10T09:28:05,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.4180 sec 2024-11-10T09:28:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:05,947 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:28:05,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-10T09:28:05,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102414b66a80e64bb3ad5b9ac4439c82bd_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230885946/Put/seqid=0 2024-11-10T09:28:05,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742381_1557 (size=17284) 2024-11-10T09:28:05,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:28:05,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102414b66a80e64bb3ad5b9ac4439c82bd_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102414b66a80e64bb3ad5b9ac4439c82bd_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:05,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/dcf48c391a3b4f40ba97e5f1d3a17ab8, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:05,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/dcf48c391a3b4f40ba97e5f1d3a17ab8 is 175, key is test_row_0/A:col10/1731230885946/Put/seqid=0 2024-11-10T09:28:05,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742382_1558 (size=48389) 2024-11-10T09:28:05,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/dcf48c391a3b4f40ba97e5f1d3a17ab8 2024-11-10T09:28:05,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/b86ec8308a854038b12f1a838f09fb3e is 50, key is test_row_0/B:col10/1731230885946/Put/seqid=0 2024-11-10T09:28:05,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742383_1559 (size=12151) 2024-11-10T09:28:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230945970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230945970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230945971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:05,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230945976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230946086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230946086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230946086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230946086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230946296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230946296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230946296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230946296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,357 INFO [Thread-2217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2217. 2024-11-10T09:28:06,357 DEBUG [Thread-2217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:06,357 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,357 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:06,358 INFO [Thread-2223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2223. 2024-11-10T09:28:06,358 DEBUG [Thread-2223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:06,358 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,358 INFO [Thread-2221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2221. 2024-11-10T09:28:06,358 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,358 DEBUG [Thread-2221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:06,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:06,358 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,358 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:06,360 INFO [Thread-2219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2219. 2024-11-10T09:28:06,360 DEBUG [Thread-2219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:06,360 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,360 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:06,361 INFO [Thread-2215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2215. 2024-11-10T09:28:06,361 DEBUG [Thread-2215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:06,361 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,361 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:06,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:06,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/b86ec8308a854038b12f1a838f09fb3e 2024-11-10T09:28:06,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/434ae48e72c242559e016fc963b9e6a0 is 50, key is test_row_0/C:col10/1731230885946/Put/seqid=0 2024-11-10T09:28:06,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742384_1560 (size=12151) 2024-11-10T09:28:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51556 deadline: 1731230946606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51564 deadline: 1731230946606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51554 deadline: 1731230946607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T09:28:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32883 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51602 deadline: 1731230946607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 2024-11-10T09:28:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 , the old value is region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T09:28:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., hostname=802346a334f5,32883,1731230704436, seqNum=5 because the exception is null or not the one we care about 2024-11-10T09:28:06,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/434ae48e72c242559e016fc963b9e6a0 2024-11-10T09:28:06,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/dcf48c391a3b4f40ba97e5f1d3a17ab8 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/dcf48c391a3b4f40ba97e5f1d3a17ab8 2024-11-10T09:28:06,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/dcf48c391a3b4f40ba97e5f1d3a17ab8, entries=250, sequenceid=252, filesize=47.3 K 2024-11-10T09:28:06,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/b86ec8308a854038b12f1a838f09fb3e as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/b86ec8308a854038b12f1a838f09fb3e 2024-11-10T09:28:06,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/b86ec8308a854038b12f1a838f09fb3e, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T09:28:06,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/434ae48e72c242559e016fc963b9e6a0 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/434ae48e72c242559e016fc963b9e6a0 2024-11-10T09:28:06,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/434ae48e72c242559e016fc963b9e6a0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-10T09:28:06,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for dedf9887fe3cb3c93b7ecf99bd810cef in 858ms, sequenceid=252, compaction requested=true 2024-11-10T09:28:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T09:28:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:06,805 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:28:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T09:28:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:06,805 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:28:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dedf9887fe3cb3c93b7ecf99bd810cef:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T09:28:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:28:06,806 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111111 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:28:06,806 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:28:06,806 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/A is initiating minor compaction (all files) 2024-11-10T09:28:06,806 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/B is initiating minor compaction (all files) 2024-11-10T09:28:06,806 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/A in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:06,806 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/B in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:06,806 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/8a252673245c48c2b1a79b3b8121b4f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5dc24e008df4423984afef3080b2ebab, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/b86ec8308a854038b12f1a838f09fb3e] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=36.1 K 2024-11-10T09:28:06,806 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/231ac36339ed402b9e478cc08ad3a14b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0ceb269d9f924f4495b4e1853cf37f0f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/dcf48c391a3b4f40ba97e5f1d3a17ab8] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=108.5 K 2024-11-10T09:28:06,806 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:06,806 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. files: [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/231ac36339ed402b9e478cc08ad3a14b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0ceb269d9f924f4495b4e1853cf37f0f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/dcf48c391a3b4f40ba97e5f1d3a17ab8] 2024-11-10T09:28:06,806 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a252673245c48c2b1a79b3b8121b4f8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230880459 2024-11-10T09:28:06,807 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 231ac36339ed402b9e478cc08ad3a14b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230880459 2024-11-10T09:28:06,807 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc24e008df4423984afef3080b2ebab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731230882636 2024-11-10T09:28:06,807 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ceb269d9f924f4495b4e1853cf37f0f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731230882636 2024-11-10T09:28:06,807 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf48c391a3b4f40ba97e5f1d3a17ab8, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230884805 2024-11-10T09:28:06,807 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting b86ec8308a854038b12f1a838f09fb3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230884810 2024-11-10T09:28:06,811 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:06,812 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:06,812 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/f249e033c0834817ab4de7f63c52b4db is 50, key is test_row_0/B:col10/1731230885946/Put/seqid=0 2024-11-10T09:28:06,812 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411106713b53eb57e4e428e253c21c1ad90c7_dedf9887fe3cb3c93b7ecf99bd810cef store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:06,816 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411106713b53eb57e4e428e253c21c1ad90c7_dedf9887fe3cb3c93b7ecf99bd810cef, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:06,816 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106713b53eb57e4e428e253c21c1ad90c7_dedf9887fe3cb3c93b7ecf99bd810cef because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:06,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742385_1561 (size=12765) 2024-11-10T09:28:06,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742386_1562 (size=4469) 2024-11-10T09:28:07,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32883 {}] regionserver.HRegion(8855): Flush requested on dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:07,119 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T09:28:07,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-10T09:28:07,119 INFO [Thread-2204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2204. 2024-11-10T09:28:07,119 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-11-10T09:28:07,119 INFO [Thread-2208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2208. 2024-11-10T09:28:07,119 DEBUG [Thread-2204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:07,120 DEBUG [Thread-2208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:07,120 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:07,120 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,120 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,120 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,120 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,120 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:07,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:07,120 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:07,122 INFO [Thread-2210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2210. 2024-11-10T09:28:07,122 DEBUG [Thread-2210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:07,122 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,123 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:07,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:07,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c2defda9672445ddad5da5fd01768e83_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_0/A:col10/1731230885969/Put/seqid=0 2024-11-10T09:28:07,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742387_1563 (size=12454) 2024-11-10T09:28:07,221 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:07,222 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/069877a12fd2407eb2ccfc3172545f82 is 175, key is test_row_0/A:col10/1731230885946/Put/seqid=0 2024-11-10T09:28:07,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742388_1564 (size=31719) 2024-11-10T09:28:07,228 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/f249e033c0834817ab4de7f63c52b4db as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/f249e033c0834817ab4de7f63c52b4db 2024-11-10T09:28:07,233 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/B of dedf9887fe3cb3c93b7ecf99bd810cef into f249e033c0834817ab4de7f63c52b4db(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:07,233 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:07,233 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/B, priority=13, startTime=1731230886805; duration=0sec 2024-11-10T09:28:07,233 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T09:28:07,233 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:B 2024-11-10T09:28:07,233 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T09:28:07,234 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T09:28:07,234 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1541): dedf9887fe3cb3c93b7ecf99bd810cef/C is initiating minor compaction (all files) 2024-11-10T09:28:07,234 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dedf9887fe3cb3c93b7ecf99bd810cef/C in TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:07,234 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/43e6ca91bd5549c289ef1c0c483ce813, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/ccda31123231495e9b21f5cfb1c3a3fd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/434ae48e72c242559e016fc963b9e6a0] into tmpdir=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp, totalSize=36.1 K 2024-11-10T09:28:07,235 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e6ca91bd5549c289ef1c0c483ce813, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731230880459 2024-11-10T09:28:07,235 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting ccda31123231495e9b21f5cfb1c3a3fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731230882636 2024-11-10T09:28:07,235 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] compactions.Compactor(225): Compacting 434ae48e72c242559e016fc963b9e6a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731230884810 2024-11-10T09:28:07,240 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dedf9887fe3cb3c93b7ecf99bd810cef#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T09:28:07,241 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/51f2a266790746a7b030727fe68dc864 is 50, key is test_row_0/C:col10/1731230885946/Put/seqid=0 2024-11-10T09:28:07,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742389_1565 (size=12765) 2024-11-10T09:28:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T09:28:07,306 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:28:07,533 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:28:07,541 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c2defda9672445ddad5da5fd01768e83_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c2defda9672445ddad5da5fd01768e83_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:07,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/7e5e5f9b9860463890e690f235c41987, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:07,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/7e5e5f9b9860463890e690f235c41987 is 175, key is test_row_0/A:col10/1731230885969/Put/seqid=0 2024-11-10T09:28:07,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742390_1566 (size=31255) 2024-11-10T09:28:07,640 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/069877a12fd2407eb2ccfc3172545f82 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/069877a12fd2407eb2ccfc3172545f82 2024-11-10T09:28:07,648 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/A of dedf9887fe3cb3c93b7ecf99bd810cef into 069877a12fd2407eb2ccfc3172545f82(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:07,648 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:07,648 INFO [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/A, priority=13, startTime=1731230886805; duration=0sec 2024-11-10T09:28:07,649 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:07,649 DEBUG [RS:0;802346a334f5:32883-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:A 2024-11-10T09:28:07,649 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/51f2a266790746a7b030727fe68dc864 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/51f2a266790746a7b030727fe68dc864 2024-11-10T09:28:07,654 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dedf9887fe3cb3c93b7ecf99bd810cef/C of dedf9887fe3cb3c93b7ecf99bd810cef into 51f2a266790746a7b030727fe68dc864(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T09:28:07,654 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:07,654 INFO [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., storeName=dedf9887fe3cb3c93b7ecf99bd810cef/C, priority=13, startTime=1731230886805; duration=0sec 2024-11-10T09:28:07,654 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T09:28:07,654 DEBUG [RS:0;802346a334f5:32883-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dedf9887fe3cb3c93b7ecf99bd810cef:C 2024-11-10T09:28:07,948 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/7e5e5f9b9860463890e690f235c41987 2024-11-10T09:28:07,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/0bd363af70d640d79cc0af7f0ed7d50f is 50, key is test_row_0/B:col10/1731230885969/Put/seqid=0 2024-11-10T09:28:07,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742391_1567 (size=12301) 2024-11-10T09:28:07,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/0bd363af70d640d79cc0af7f0ed7d50f 2024-11-10T09:28:07,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/4fa248a936ec4e1d94a8cf4489e5dabf is 50, key is test_row_0/C:col10/1731230885969/Put/seqid=0 2024-11-10T09:28:07,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742392_1568 (size=12301) 2024-11-10T09:28:08,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/4fa248a936ec4e1d94a8cf4489e5dabf 2024-11-10T09:28:08,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/7e5e5f9b9860463890e690f235c41987 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7e5e5f9b9860463890e690f235c41987 2024-11-10T09:28:08,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7e5e5f9b9860463890e690f235c41987, entries=150, sequenceid=275, filesize=30.5 K 2024-11-10T09:28:08,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/0bd363af70d640d79cc0af7f0ed7d50f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/0bd363af70d640d79cc0af7f0ed7d50f 2024-11-10T09:28:08,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/0bd363af70d640d79cc0af7f0ed7d50f, entries=150, sequenceid=275, filesize=12.0 K 2024-11-10T09:28:08,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/4fa248a936ec4e1d94a8cf4489e5dabf as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4fa248a936ec4e1d94a8cf4489e5dabf 2024-11-10T09:28:08,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4fa248a936ec4e1d94a8cf4489e5dabf, entries=150, sequenceid=275, filesize=12.0 K 2024-11-10T09:28:08,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=6.71 KB/6870 for dedf9887fe3cb3c93b7ecf99bd810cef in 1290ms, sequenceid=275, compaction requested=false 2024-11-10T09:28:08,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dedf9887fe3cb3c93b7ecf99bd810cef: 2024-11-10T09:28:12,929 INFO [Thread-2206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2206. 2024-11-10T09:28:12,929 DEBUG [Thread-2206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T09:28:12,929 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:12,930 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:12,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:12,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T09:28:12,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-10T09:28:12,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-10T09:28:12,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-10T09:28:12,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-10T09:28:12,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-10T09:28:12,930 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T09:28:12,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4463 2024-11-10T09:28:12,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4607 2024-11-10T09:28:12,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4425 2024-11-10T09:28:12,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4467 2024-11-10T09:28:12,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4616 2024-11-10T09:28:12,931 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T09:28:12,931 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:28:12,931 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:28:12,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:12,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:12,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:12,933 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-10T09:28:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T09:28:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:28:12,939 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230892939"}]},"ts":"1731230892939"} 2024-11-10T09:28:12,941 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T09:28:12,941 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T09:28:12,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T09:28:12,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, UNASSIGN}] 2024-11-10T09:28:12,946 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, UNASSIGN 2024-11-10T09:28:12,947 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=CLOSING, regionLocation=802346a334f5,32883,1731230704436 2024-11-10T09:28:12,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, UNASSIGN because future has completed 2024-11-10T09:28:12,949 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T09:28:12,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436}] 2024-11-10T09:28:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:28:13,102 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:13,102 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T09:28:13,102 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing dedf9887fe3cb3c93b7ecf99bd810cef, disabling compactions & flushes 2024-11-10T09:28:13,102 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:13,102 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:13,102 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. after waiting 0 ms 2024-11-10T09:28:13,102 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:13,102 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing dedf9887fe3cb3c93b7ecf99bd810cef 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-10T09:28:13,107 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106f574def8bea41078d774e4b6ce89365_dedf9887fe3cb3c93b7ecf99bd810cef is 50, key is test_row_1/A:col10/1731230892926/Put/seqid=0 2024-11-10T09:28:13,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742393_1569 (size=7374) 2024-11-10T09:28:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:28:13,513 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T09:28:13,523 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106f574def8bea41078d774e4b6ce89365_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106f574def8bea41078d774e4b6ce89365_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:13,524 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/ee87fba2c7bf4738bf4209d4c1f769d3, store: [table=TestAcidGuarantees family=A region=dedf9887fe3cb3c93b7ecf99bd810cef] 2024-11-10T09:28:13,525 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/ee87fba2c7bf4738bf4209d4c1f769d3 is 175, key is test_row_1/A:col10/1731230892926/Put/seqid=0 2024-11-10T09:28:13,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742394_1570 (size=13865) 2024-11-10T09:28:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:28:13,932 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/ee87fba2c7bf4738bf4209d4c1f769d3 2024-11-10T09:28:13,944 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/92cecfa89a7f43de90b85ee740bf228f is 50, key is test_row_1/B:col10/1731230892926/Put/seqid=0 2024-11-10T09:28:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742395_1571 (size=7415) 2024-11-10T09:28:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:28:14,376 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/92cecfa89a7f43de90b85ee740bf228f 2024-11-10T09:28:14,387 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/75202aac8d864ccd9c81421223dee3c3 is 50, key is test_row_1/C:col10/1731230892926/Put/seqid=0 2024-11-10T09:28:14,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742396_1572 (size=7415) 2024-11-10T09:28:14,793 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/75202aac8d864ccd9c81421223dee3c3 2024-11-10T09:28:14,803 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/A/ee87fba2c7bf4738bf4209d4c1f769d3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/ee87fba2c7bf4738bf4209d4c1f769d3 2024-11-10T09:28:14,807 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/ee87fba2c7bf4738bf4209d4c1f769d3, entries=50, sequenceid=283, filesize=13.5 K 2024-11-10T09:28:14,808 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/B/92cecfa89a7f43de90b85ee740bf228f as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/92cecfa89a7f43de90b85ee740bf228f 2024-11-10T09:28:14,812 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/92cecfa89a7f43de90b85ee740bf228f, entries=50, sequenceid=283, filesize=7.2 K 2024-11-10T09:28:14,813 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/.tmp/C/75202aac8d864ccd9c81421223dee3c3 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/75202aac8d864ccd9c81421223dee3c3 2024-11-10T09:28:14,817 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/75202aac8d864ccd9c81421223dee3c3, entries=50, sequenceid=283, filesize=7.2 K 2024-11-10T09:28:14,817 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for dedf9887fe3cb3c93b7ecf99bd810cef in 1715ms, sequenceid=283, compaction requested=true 2024-11-10T09:28:14,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/e042482d642d4171bfd135fc17e07d27, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/908985dbfa814569894cf61656df5ee6, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/634d62a6ebdb4bbb9d1510c0f474d188, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/f88e45a00ca44f5eb1304600e087f287, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/666341ecca2a415c8327785dcda73f16, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4336c7226be34f839573bf55990b3f2d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/18ca409e386e429a9a118c87353f1057, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4fcf2c3f89a64cf48c9b4f6a01f78af7, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0b0ff027453f486583a2709546b3b2d1, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/69ed34a7ddb5406ea244987491160c48, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/a9fd586dc161490d9e11d60dca7ba38b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7a625b980ccf40aca46095ae4e7745fe, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/26579e00f80c43f4871ce120a6e98b7d, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/333467eaf5a14355ae5cd15e9accfe4c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/2b25721eba614ffd84ece9a573f5da9c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/231ac36339ed402b9e478cc08ad3a14b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0ceb269d9f924f4495b4e1853cf37f0f, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/dcf48c391a3b4f40ba97e5f1d3a17ab8] to archive 2024-11-10T09:28:14,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:28:14,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/e042482d642d4171bfd135fc17e07d27 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/e042482d642d4171bfd135fc17e07d27 2024-11-10T09:28:14,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/908985dbfa814569894cf61656df5ee6 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/908985dbfa814569894cf61656df5ee6 2024-11-10T09:28:14,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/634d62a6ebdb4bbb9d1510c0f474d188 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/634d62a6ebdb4bbb9d1510c0f474d188 2024-11-10T09:28:14,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/f88e45a00ca44f5eb1304600e087f287 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/f88e45a00ca44f5eb1304600e087f287 2024-11-10T09:28:14,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/666341ecca2a415c8327785dcda73f16 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/666341ecca2a415c8327785dcda73f16 2024-11-10T09:28:14,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4336c7226be34f839573bf55990b3f2d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4336c7226be34f839573bf55990b3f2d 2024-11-10T09:28:14,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/18ca409e386e429a9a118c87353f1057 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/18ca409e386e429a9a118c87353f1057 2024-11-10T09:28:14,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4fcf2c3f89a64cf48c9b4f6a01f78af7 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/4fcf2c3f89a64cf48c9b4f6a01f78af7 2024-11-10T09:28:14,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0b0ff027453f486583a2709546b3b2d1 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0b0ff027453f486583a2709546b3b2d1 2024-11-10T09:28:14,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/69ed34a7ddb5406ea244987491160c48 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/69ed34a7ddb5406ea244987491160c48 2024-11-10T09:28:14,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/a9fd586dc161490d9e11d60dca7ba38b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/a9fd586dc161490d9e11d60dca7ba38b 2024-11-10T09:28:14,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7a625b980ccf40aca46095ae4e7745fe to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7a625b980ccf40aca46095ae4e7745fe 2024-11-10T09:28:14,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/26579e00f80c43f4871ce120a6e98b7d to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/26579e00f80c43f4871ce120a6e98b7d 2024-11-10T09:28:14,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/333467eaf5a14355ae5cd15e9accfe4c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/333467eaf5a14355ae5cd15e9accfe4c 2024-11-10T09:28:14,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/2b25721eba614ffd84ece9a573f5da9c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/2b25721eba614ffd84ece9a573f5da9c 2024-11-10T09:28:14,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/231ac36339ed402b9e478cc08ad3a14b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/231ac36339ed402b9e478cc08ad3a14b 2024-11-10T09:28:14,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0ceb269d9f924f4495b4e1853cf37f0f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/0ceb269d9f924f4495b4e1853cf37f0f 2024-11-10T09:28:14,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/dcf48c391a3b4f40ba97e5f1d3a17ab8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/dcf48c391a3b4f40ba97e5f1d3a17ab8 2024-11-10T09:28:14,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/961a7a287c9f47fe82c5b7c96d189f4b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/3f15da88c5094620a31c75af898a45ef, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/27cc3749728141a1aa68e0c59f3de8ac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/2d9c72beab60417980a5c43245c8c645, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/53639f65c0f5429daa8446f4b41fa9e4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/a7606afbbda64fc78097d4a7a2616bb2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/96a60c8c73304b599b0454f1357bc377, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/01321d2a3f294ae1bdb9ea37ba9a1709, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/7058cc72939749c5834bc22a9f247bac, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/4d9b933a332d41aebcf8882efd26d004, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/30523d4cfa8e4245957ec785485e4e3a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/47dde69a66b4460eb03b06a511664c44, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/9c32aba15f224dda85c66710b9baf1f2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5b604bac8975408383359a19fd17f1f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/8a252673245c48c2b1a79b3b8121b4f8, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/e11ec53a2e4241a28346465d2b7c0c26, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5dc24e008df4423984afef3080b2ebab, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/b86ec8308a854038b12f1a838f09fb3e] to archive 2024-11-10T09:28:14,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:28:14,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/961a7a287c9f47fe82c5b7c96d189f4b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/961a7a287c9f47fe82c5b7c96d189f4b 2024-11-10T09:28:14,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/3f15da88c5094620a31c75af898a45ef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/3f15da88c5094620a31c75af898a45ef 2024-11-10T09:28:14,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/27cc3749728141a1aa68e0c59f3de8ac to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/27cc3749728141a1aa68e0c59f3de8ac 2024-11-10T09:28:14,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/2d9c72beab60417980a5c43245c8c645 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/2d9c72beab60417980a5c43245c8c645 2024-11-10T09:28:14,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/53639f65c0f5429daa8446f4b41fa9e4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/53639f65c0f5429daa8446f4b41fa9e4 2024-11-10T09:28:14,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/a7606afbbda64fc78097d4a7a2616bb2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/a7606afbbda64fc78097d4a7a2616bb2 2024-11-10T09:28:14,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/96a60c8c73304b599b0454f1357bc377 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/96a60c8c73304b599b0454f1357bc377 2024-11-10T09:28:14,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/01321d2a3f294ae1bdb9ea37ba9a1709 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/01321d2a3f294ae1bdb9ea37ba9a1709 2024-11-10T09:28:14,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/7058cc72939749c5834bc22a9f247bac to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/7058cc72939749c5834bc22a9f247bac 2024-11-10T09:28:14,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/4d9b933a332d41aebcf8882efd26d004 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/4d9b933a332d41aebcf8882efd26d004 2024-11-10T09:28:14,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/30523d4cfa8e4245957ec785485e4e3a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/30523d4cfa8e4245957ec785485e4e3a 2024-11-10T09:28:14,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/47dde69a66b4460eb03b06a511664c44 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/47dde69a66b4460eb03b06a511664c44 2024-11-10T09:28:14,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/9c32aba15f224dda85c66710b9baf1f2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/9c32aba15f224dda85c66710b9baf1f2 2024-11-10T09:28:14,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5b604bac8975408383359a19fd17f1f8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5b604bac8975408383359a19fd17f1f8 2024-11-10T09:28:14,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/8a252673245c48c2b1a79b3b8121b4f8 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/8a252673245c48c2b1a79b3b8121b4f8 2024-11-10T09:28:14,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/e11ec53a2e4241a28346465d2b7c0c26 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/e11ec53a2e4241a28346465d2b7c0c26 2024-11-10T09:28:14,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5dc24e008df4423984afef3080b2ebab to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/5dc24e008df4423984afef3080b2ebab 2024-11-10T09:28:14,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/b86ec8308a854038b12f1a838f09fb3e to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/b86ec8308a854038b12f1a838f09fb3e 2024-11-10T09:28:14,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d2b77755c8be47599fb473a408881afc, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/247a3efb89ee42119d19ebf048dc0fae, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/2a696426664748d989f16606168e467b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/0f948ce8e2484cf3a801ef34366adcc4, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d101ce5dfa36430c9766c7abbe59f126, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a0c37cb5450d41b5aafca8ce1096ece2, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/38bb381c0dd54e3db70d3ca59e8f915b, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/1aa1caa057bb460786d061751af56980, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/8e70992c2df2470f90b5dfa3b01b0cc9, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4ca58bfc0bfa4268ae22cd6ed1a9123a, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/41b137e2c7d84c00a903370a81ea629c, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/b910a105f6a045beb4c6c36387976d41, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a5f8949a416e4cee823ead8604151400, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/df2c46071fee4c288622e4f0e49d6598, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/43e6ca91bd5549c289ef1c0c483ce813, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/cbd3fcfbbaf448c2a0ff4deab429cde5, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/ccda31123231495e9b21f5cfb1c3a3fd, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/434ae48e72c242559e016fc963b9e6a0] to archive 2024-11-10T09:28:14,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T09:28:14,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d2b77755c8be47599fb473a408881afc to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d2b77755c8be47599fb473a408881afc 2024-11-10T09:28:14,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/247a3efb89ee42119d19ebf048dc0fae to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/247a3efb89ee42119d19ebf048dc0fae 2024-11-10T09:28:14,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/2a696426664748d989f16606168e467b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/2a696426664748d989f16606168e467b 2024-11-10T09:28:14,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/0f948ce8e2484cf3a801ef34366adcc4 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/0f948ce8e2484cf3a801ef34366adcc4 2024-11-10T09:28:14,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d101ce5dfa36430c9766c7abbe59f126 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/d101ce5dfa36430c9766c7abbe59f126 2024-11-10T09:28:14,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a0c37cb5450d41b5aafca8ce1096ece2 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a0c37cb5450d41b5aafca8ce1096ece2 2024-11-10T09:28:14,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/38bb381c0dd54e3db70d3ca59e8f915b to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/38bb381c0dd54e3db70d3ca59e8f915b 2024-11-10T09:28:14,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/1aa1caa057bb460786d061751af56980 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/1aa1caa057bb460786d061751af56980 2024-11-10T09:28:14,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/8e70992c2df2470f90b5dfa3b01b0cc9 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/8e70992c2df2470f90b5dfa3b01b0cc9 2024-11-10T09:28:14,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4ca58bfc0bfa4268ae22cd6ed1a9123a to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4ca58bfc0bfa4268ae22cd6ed1a9123a 2024-11-10T09:28:14,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/41b137e2c7d84c00a903370a81ea629c to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/41b137e2c7d84c00a903370a81ea629c 2024-11-10T09:28:14,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/b910a105f6a045beb4c6c36387976d41 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/b910a105f6a045beb4c6c36387976d41 2024-11-10T09:28:14,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a5f8949a416e4cee823ead8604151400 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/a5f8949a416e4cee823ead8604151400 2024-11-10T09:28:14,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/df2c46071fee4c288622e4f0e49d6598 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/df2c46071fee4c288622e4f0e49d6598 2024-11-10T09:28:14,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/43e6ca91bd5549c289ef1c0c483ce813 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/43e6ca91bd5549c289ef1c0c483ce813 2024-11-10T09:28:14,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/cbd3fcfbbaf448c2a0ff4deab429cde5 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/cbd3fcfbbaf448c2a0ff4deab429cde5 2024-11-10T09:28:14,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/ccda31123231495e9b21f5cfb1c3a3fd to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/ccda31123231495e9b21f5cfb1c3a3fd 2024-11-10T09:28:14,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/434ae48e72c242559e016fc963b9e6a0 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/434ae48e72c242559e016fc963b9e6a0 2024-11-10T09:28:14,895 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/recovered.edits/286.seqid, newMaxSeqId=286, maxSeqId=4 2024-11-10T09:28:14,896 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. 2024-11-10T09:28:14,896 DEBUG [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for dedf9887fe3cb3c93b7ecf99bd810cef: Waiting for close lock at 1731230893102Running coprocessor pre-close hooks at 1731230893102Disabling compacts and flushes for region at 1731230893102Disabling writes for close at 1731230893102Obtaining lock to block concurrent updates at 1731230893102Preparing flush snapshotting stores in dedf9887fe3cb3c93b7ecf99bd810cef at 1731230893102Finished memstore snapshotting TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731230893102Flushing stores of TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef. at 1731230893103 (+1 ms)Flushing dedf9887fe3cb3c93b7ecf99bd810cef/A: creating writer at 1731230893103Flushing mob file dedf9887fe3cb3c93b7ecf99bd810cef/A: appending metadata at 1731230893107 (+4 ms)Flushing mob file dedf9887fe3cb3c93b7ecf99bd810cef/A: closing flushed file at 1731230893107Flushing dedf9887fe3cb3c93b7ecf99bd810cef/A: appending metadata at 1731230893524 (+417 ms)Flushing dedf9887fe3cb3c93b7ecf99bd810cef/A: closing flushed file at 1731230893524Flushing dedf9887fe3cb3c93b7ecf99bd810cef/B: creating writer at 1731230893940 (+416 ms)Flushing dedf9887fe3cb3c93b7ecf99bd810cef/B: appending metadata at 1731230893944 (+4 ms)Flushing dedf9887fe3cb3c93b7ecf99bd810cef/B: closing flushed file at 1731230893944Flushing dedf9887fe3cb3c93b7ecf99bd810cef/C: creating writer at 1731230894383 (+439 ms)Flushing dedf9887fe3cb3c93b7ecf99bd810cef/C: appending metadata at 1731230894386 (+3 ms)Flushing dedf9887fe3cb3c93b7ecf99bd810cef/C: closing flushed file at 1731230894386Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b958750: reopening flushed file at 1731230894802 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2db0ec05: reopening flushed file at 1731230894807 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4066cb62: reopening flushed file at 1731230894812 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for dedf9887fe3cb3c93b7ecf99bd810cef in 1715ms, sequenceid=283, compaction requested=true at 1731230894817 (+5 ms)Writing region close event to WAL at 1731230894893 (+76 ms)Running coprocessor post-close hooks at 1731230894896 (+3 ms)Closed at 1731230894896 2024-11-10T09:28:14,898 INFO [RS_CLOSE_REGION-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:14,899 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=dedf9887fe3cb3c93b7ecf99bd810cef, regionState=CLOSED 2024-11-10T09:28:14,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 because future has completed 2024-11-10T09:28:14,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-10T09:28:14,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure dedf9887fe3cb3c93b7ecf99bd810cef, server=802346a334f5,32883,1731230704436 in 1.9520 sec 2024-11-10T09:28:14,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-10T09:28:14,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dedf9887fe3cb3c93b7ecf99bd810cef, UNASSIGN in 1.9590 sec 2024-11-10T09:28:14,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-10T09:28:14,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9630 sec 2024-11-10T09:28:14,909 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731230894909"}]},"ts":"1731230894909"} 2024-11-10T09:28:14,910 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T09:28:14,910 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T09:28:14,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9770 sec 2024-11-10T09:28:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T09:28:15,097 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:28:15,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-10T09:28:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:28:15,104 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:28:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T09:28:15,106 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:28:15,109 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,112 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C, FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/recovered.edits] 2024-11-10T09:28:15,115 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/069877a12fd2407eb2ccfc3172545f82 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/069877a12fd2407eb2ccfc3172545f82 2024-11-10T09:28:15,117 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7e5e5f9b9860463890e690f235c41987 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/7e5e5f9b9860463890e690f235c41987 2024-11-10T09:28:15,118 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/ee87fba2c7bf4738bf4209d4c1f769d3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/A/ee87fba2c7bf4738bf4209d4c1f769d3 2024-11-10T09:28:15,120 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/0bd363af70d640d79cc0af7f0ed7d50f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/0bd363af70d640d79cc0af7f0ed7d50f 2024-11-10T09:28:15,122 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/92cecfa89a7f43de90b85ee740bf228f to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/92cecfa89a7f43de90b85ee740bf228f 2024-11-10T09:28:15,123 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/f249e033c0834817ab4de7f63c52b4db to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/B/f249e033c0834817ab4de7f63c52b4db 2024-11-10T09:28:15,125 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4fa248a936ec4e1d94a8cf4489e5dabf to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/4fa248a936ec4e1d94a8cf4489e5dabf 2024-11-10T09:28:15,126 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/51f2a266790746a7b030727fe68dc864 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/51f2a266790746a7b030727fe68dc864 2024-11-10T09:28:15,127 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/75202aac8d864ccd9c81421223dee3c3 to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/C/75202aac8d864ccd9c81421223dee3c3 2024-11-10T09:28:15,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/recovered.edits/286.seqid to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef/recovered.edits/286.seqid 2024-11-10T09:28:15,131 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/default/TestAcidGuarantees/dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,131 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T09:28:15,131 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:28:15,132 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T09:28:15,135 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102414b66a80e64bb3ad5b9ac4439c82bd_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102414b66a80e64bb3ad5b9ac4439c82bd_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,137 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e3543ecb52d4235b22f71defe2b19a7_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103e3543ecb52d4235b22f71defe2b19a7_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,139 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111049a500a0882e492b8e327260448e47cd_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111049a500a0882e492b8e327260448e47cd_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,140 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110626c560b199c45c694f286c1f38f24a9_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110626c560b199c45c694f286c1f38f24a9_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,142 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110672ce65acbcc46479fd7f5753aca2ef6_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110672ce65acbcc46479fd7f5753aca2ef6_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,145 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106f574def8bea41078d774e4b6ce89365_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106f574def8bea41078d774e4b6ce89365_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,146 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107dfe8722414f4d37a92b7907f301f6cb_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107dfe8722414f4d37a92b7907f301f6cb_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,147 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109f73cbd435a048fc9e52cf6777dfe262_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109f73cbd435a048fc9e52cf6777dfe262_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,149 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a00bd2473d7d47afbe51f8562c5677a5_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a00bd2473d7d47afbe51f8562c5677a5_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,150 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a0c3d9c3fbe14b13853e47440d0e6dc5_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a0c3d9c3fbe14b13853e47440d0e6dc5_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,151 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bc07fd7d7b154ae3b423956a6dadd11a_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bc07fd7d7b154ae3b423956a6dadd11a_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,152 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c2defda9672445ddad5da5fd01768e83_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c2defda9672445ddad5da5fd01768e83_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,154 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c331a6cc754e44479e22ecbe67f65905_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c331a6cc754e44479e22ecbe67f65905_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,155 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c948230ca95a4134b11ca0ae1b92f31c_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c948230ca95a4134b11ca0ae1b92f31c_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,156 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f42ee98e68584bae983f3ba5dea0e02e_dedf9887fe3cb3c93b7ecf99bd810cef to hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f42ee98e68584bae983f3ba5dea0e02e_dedf9887fe3cb3c93b7ecf99bd810cef 2024-11-10T09:28:15,156 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T09:28:15,158 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:28:15,161 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T09:28:15,163 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T09:28:15,164 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:28:15,164 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T09:28:15,165 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731230895164"}]},"ts":"9223372036854775807"} 2024-11-10T09:28:15,167 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T09:28:15,167 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dedf9887fe3cb3c93b7ecf99bd810cef, NAME => 'TestAcidGuarantees,,1731230863188.dedf9887fe3cb3c93b7ecf99bd810cef.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T09:28:15,167 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T09:28:15,167 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731230895167"}]},"ts":"9223372036854775807"} 2024-11-10T09:28:15,169 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T09:28:15,169 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T09:28:15,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-10T09:28:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38235 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T09:28:15,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T09:28:15,215 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T09:28:15,226 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238 (was 240), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 350), ProcessCount=11 (was 11), AvailableMemoryMB=6811 (was 6831) 2024-11-10T09:28:15,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T09:28:15,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:28:15,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:28:15,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:15,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:15,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T09:28:15,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T09:28:15,227 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-10T09:28:15,227 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=802346a334f5,38235,1731230703274 2024-11-10T09:28:15,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T09:28:15,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T09:28:15,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:28:15,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T09:28:15,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:28:15,290 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T09:28:15,290 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:28:15,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:15,291 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '802346a334f5,32883,1731230704436' ***** 2024-11-10T09:28:15,291 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:28:15,291 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T09:28:15,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T09:28:15,292 INFO [RS:0;802346a334f5:32883 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T09:28:15,293 INFO [RS:0;802346a334f5:32883 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T09:28:15,293 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T09:28:15,293 INFO [RS:0;802346a334f5:32883 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T09:28:15,293 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(959): stopping server 802346a334f5,32883,1731230704436 2024-11-10T09:28:15,294 INFO [RS:0;802346a334f5:32883 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T09:28:15,294 INFO [RS:0;802346a334f5:32883 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;802346a334f5:32883. 2024-11-10T09:28:15,294 DEBUG [RS:0;802346a334f5:32883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T09:28:15,294 DEBUG [RS:0;802346a334f5:32883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:15,294 INFO [RS:0;802346a334f5:32883 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T09:28:15,294 INFO [RS:0;802346a334f5:32883 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T09:28:15,294 INFO [RS:0;802346a334f5:32883 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T09:28:15,294 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T09:28:15,295 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T09:28:15,296 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T09:28:15,296 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:15,296 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T09:28:15,296 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T09:28:15,296 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T09:28:15,296 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T09:28:15,296 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T09:28:15,296 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T09:28:15,308 INFO [regionserver/802346a334f5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T09:28:15,315 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/info/41dbae1bf37e4d738f1e98d4b0b368b9 is 95, key is TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3./info:/1731230738876/DeleteFamily/seqid=0 2024-11-10T09:28:15,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742397_1573 (size=6514) 2024-11-10T09:28:15,496 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:15,500 INFO [regionserver/802346a334f5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T09:28:15,500 INFO [regionserver/802346a334f5:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T09:28:15,696 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:15,723 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/info/41dbae1bf37e4d738f1e98d4b0b368b9 2024-11-10T09:28:15,745 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/ns/020e629e678149cf884b6eada3399235 is 93, key is TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3./ns:/1731230738842/DeleteFamily/seqid=0 2024-11-10T09:28:15,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742398_1574 (size=5906) 2024-11-10T09:28:15,897 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:16,097 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:16,150 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/ns/020e629e678149cf884b6eada3399235 2024-11-10T09:28:16,175 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/rep_barrier/85e07a3b2029491ca4854dd86d7c0a23 is 102, key is TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3./rep_barrier:/1731230738842/DeleteFamily/seqid=0 2024-11-10T09:28:16,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742399_1575 (size=6025) 2024-11-10T09:28:16,297 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:16,498 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:16,580 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/rep_barrier/85e07a3b2029491ca4854dd86d7c0a23 2024-11-10T09:28:16,606 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/table/84d91d9538c0496d8139dcdb72836e62 is 96, key is TestAcidGuarantees,,1731230708965.62a21c798fe3504a706a57ba0baaaff3./table:/1731230738842/DeleteFamily/seqid=0 2024-11-10T09:28:16,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742400_1576 (size=5936) 2024-11-10T09:28:16,698 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:16,898 DEBUG [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T09:28:17,011 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/table/84d91d9538c0496d8139dcdb72836e62 2024-11-10T09:28:17,020 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/info/41dbae1bf37e4d738f1e98d4b0b368b9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/info/41dbae1bf37e4d738f1e98d4b0b368b9 2024-11-10T09:28:17,025 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/info/41dbae1bf37e4d738f1e98d4b0b368b9, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T09:28:17,026 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/ns/020e629e678149cf884b6eada3399235 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/ns/020e629e678149cf884b6eada3399235 2024-11-10T09:28:17,029 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/ns/020e629e678149cf884b6eada3399235, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T09:28:17,030 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/rep_barrier/85e07a3b2029491ca4854dd86d7c0a23 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/rep_barrier/85e07a3b2029491ca4854dd86d7c0a23 2024-11-10T09:28:17,034 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/rep_barrier/85e07a3b2029491ca4854dd86d7c0a23, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T09:28:17,035 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/.tmp/table/84d91d9538c0496d8139dcdb72836e62 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/table/84d91d9538c0496d8139dcdb72836e62 2024-11-10T09:28:17,038 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/table/84d91d9538c0496d8139dcdb72836e62, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T09:28:17,039 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false 2024-11-10T09:28:17,043 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T09:28:17,044 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T09:28:17,044 INFO [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T09:28:17,044 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731230895295Running coprocessor pre-close hooks at 1731230895296 (+1 ms)Disabling compacts and flushes for region at 1731230895296Disabling writes for close at 1731230895296Obtaining lock to block concurrent updates at 1731230895296Preparing flush snapshotting stores in 1588230740 at 1731230895296Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731230895297 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731230895297Flushing 1588230740/info: creating writer at 1731230895298 (+1 ms)Flushing 1588230740/info: appending metadata at 1731230895315 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731230895315Flushing 1588230740/ns: creating writer at 1731230895733 (+418 ms)Flushing 1588230740/ns: appending metadata at 1731230895745 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1731230895745Flushing 1588230740/rep_barrier: creating writer at 1731230896158 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731230896175 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731230896175Flushing 1588230740/table: creating writer at 1731230896589 (+414 ms)Flushing 1588230740/table: appending metadata at 1731230896606 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731230896606Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39cb9f: reopening flushed file at 1731230897020 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78801331: reopening flushed file at 1731230897025 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c959230: reopening flushed file at 1731230897030 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cb4506a: reopening flushed file at 1731230897034 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false at 1731230897039 (+5 ms)Writing region close event to WAL at 1731230897040 (+1 ms)Running coprocessor post-close hooks at 1731230897043 (+3 ms)Closed at 1731230897044 (+1 ms) 2024-11-10T09:28:17,044 DEBUG [RS_CLOSE_META-regionserver/802346a334f5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T09:28:17,099 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(976): stopping server 802346a334f5,32883,1731230704436; all regions closed. 2024-11-10T09:28:17,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741834_1010 (size=25344) 2024-11-10T09:28:17,110 DEBUG [RS:0;802346a334f5:32883 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/oldWALs 2024-11-10T09:28:17,110 INFO [RS:0;802346a334f5:32883 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 802346a334f5%2C32883%2C1731230704436.meta:.meta(num 1731230708301) 2024-11-10T09:28:17,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741833_1009 (size=13065607) 2024-11-10T09:28:17,114 DEBUG [RS:0;802346a334f5:32883 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/oldWALs 2024-11-10T09:28:17,114 INFO [RS:0;802346a334f5:32883 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 802346a334f5%2C32883%2C1731230704436:(num 1731230707649) 2024-11-10T09:28:17,114 DEBUG [RS:0;802346a334f5:32883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T09:28:17,114 INFO [RS:0;802346a334f5:32883 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T09:28:17,114 INFO [RS:0;802346a334f5:32883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T09:28:17,115 INFO [RS:0;802346a334f5:32883 {}] hbase.ChoreService(370): Chore service for: regionserver/802346a334f5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T09:28:17,115 INFO [RS:0;802346a334f5:32883 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T09:28:17,115 INFO [regionserver/802346a334f5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T09:28:17,115 INFO [RS:0;802346a334f5:32883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:32883 2024-11-10T09:28:17,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/802346a334f5,32883,1731230704436 2024-11-10T09:28:17,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T09:28:17,162 INFO [RS:0;802346a334f5:32883 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T09:28:17,163 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [802346a334f5,32883,1731230704436] 2024-11-10T09:28:17,182 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/802346a334f5,32883,1731230704436 already deleted, retry=false 2024-11-10T09:28:17,183 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 802346a334f5,32883,1731230704436 expired; onlineServers=0 2024-11-10T09:28:17,183 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '802346a334f5,38235,1731230703274' ***** 2024-11-10T09:28:17,183 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T09:28:17,183 INFO [M:0;802346a334f5:38235 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T09:28:17,183 INFO [M:0;802346a334f5:38235 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T09:28:17,184 DEBUG [M:0;802346a334f5:38235 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T09:28:17,184 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T09:28:17,184 DEBUG [M:0;802346a334f5:38235 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T09:28:17,184 DEBUG [master/802346a334f5:0:becomeActiveMaster-HFileCleaner.small.0-1731230707364 {}] cleaner.HFileCleaner(306): Exit Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.small.0-1731230707364,5,FailOnTimeoutGroup] 2024-11-10T09:28:17,184 DEBUG [master/802346a334f5:0:becomeActiveMaster-HFileCleaner.large.0-1731230707358 {}] cleaner.HFileCleaner(306): Exit Thread[master/802346a334f5:0:becomeActiveMaster-HFileCleaner.large.0-1731230707358,5,FailOnTimeoutGroup] 2024-11-10T09:28:17,184 INFO [M:0;802346a334f5:38235 {}] hbase.ChoreService(370): Chore service for: master/802346a334f5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T09:28:17,185 INFO [M:0;802346a334f5:38235 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T09:28:17,185 DEBUG [M:0;802346a334f5:38235 {}] master.HMaster(1795): Stopping service threads 2024-11-10T09:28:17,185 INFO [M:0;802346a334f5:38235 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T09:28:17,185 INFO [M:0;802346a334f5:38235 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T09:28:17,186 ERROR [M:0;802346a334f5:38235 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (879580466) connection to localhost/127.0.0.1:37473 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37473,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T09:28:17,187 INFO [M:0;802346a334f5:38235 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T09:28:17,187 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T09:28:17,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T09:28:17,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T09:28:17,193 DEBUG [M:0;802346a334f5:38235 {}] zookeeper.ZKUtil(347): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T09:28:17,193 WARN [M:0;802346a334f5:38235 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T09:28:17,194 INFO [M:0;802346a334f5:38235 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/.lastflushedseqids 2024-11-10T09:28:17,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742401_1577 (size=72) 2024-11-10T09:28:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:28:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32883-0x101241016650001, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:28:17,274 INFO [RS:0;802346a334f5:32883 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T09:28:17,274 INFO [RS:0;802346a334f5:32883 {}] regionserver.HRegionServer(1031): Exiting; stopping=802346a334f5,32883,1731230704436; zookeeper connection closed. 2024-11-10T09:28:17,275 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5efdd944 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5efdd944 2024-11-10T09:28:17,276 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T09:28:17,606 INFO [M:0;802346a334f5:38235 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T09:28:17,607 INFO [M:0;802346a334f5:38235 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T09:28:17,607 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T09:28:17,607 INFO [M:0;802346a334f5:38235 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:28:17,607 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:28:17,608 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T09:28:17,608 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:28:17,608 INFO [M:0;802346a334f5:38235 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=732.96 KB heapSize=899.24 KB 2024-11-10T09:28:17,629 DEBUG [M:0;802346a334f5:38235 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6dd85eaa9e3843dcbd53da7d84ec3fe9 is 82, key is hbase:meta,,1/info:regioninfo/1731230708420/Put/seqid=0 2024-11-10T09:28:17,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742402_1578 (size=5672) 2024-11-10T09:28:18,034 INFO [M:0;802346a334f5:38235 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2045 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6dd85eaa9e3843dcbd53da7d84ec3fe9 2024-11-10T09:28:18,059 DEBUG [M:0;802346a334f5:38235 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/855a27657ac34b96bed6681645b099b4 is 2276, key is \x00\x00\x00\x00\x00\x00\x00%/proc:d/1731230742109/Put/seqid=0 2024-11-10T09:28:18,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742403_1579 (size=43844) 2024-11-10T09:28:18,464 INFO [M:0;802346a334f5:38235 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732.41 KB at sequenceid=2045 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/855a27657ac34b96bed6681645b099b4 2024-11-10T09:28:18,474 INFO [M:0;802346a334f5:38235 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 855a27657ac34b96bed6681645b099b4 2024-11-10T09:28:18,490 DEBUG [M:0;802346a334f5:38235 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b4b6df6a202b43a7a80ff75ab03923e4 is 69, key is 802346a334f5,32883,1731230704436/rs:state/1731230707403/Put/seqid=0 2024-11-10T09:28:18,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073742404_1580 (size=5156) 2024-11-10T09:28:18,894 INFO [M:0;802346a334f5:38235 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2045 (bloomFilter=true), to=hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b4b6df6a202b43a7a80ff75ab03923e4 2024-11-10T09:28:18,903 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6dd85eaa9e3843dcbd53da7d84ec3fe9 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6dd85eaa9e3843dcbd53da7d84ec3fe9 2024-11-10T09:28:18,908 INFO [M:0;802346a334f5:38235 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6dd85eaa9e3843dcbd53da7d84ec3fe9, entries=8, sequenceid=2045, filesize=5.5 K 2024-11-10T09:28:18,909 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/855a27657ac34b96bed6681645b099b4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/855a27657ac34b96bed6681645b099b4 2024-11-10T09:28:18,912 INFO [M:0;802346a334f5:38235 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 855a27657ac34b96bed6681645b099b4 2024-11-10T09:28:18,912 INFO [M:0;802346a334f5:38235 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/855a27657ac34b96bed6681645b099b4, entries=168, sequenceid=2045, filesize=42.8 K 2024-11-10T09:28:18,913 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b4b6df6a202b43a7a80ff75ab03923e4 as hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b4b6df6a202b43a7a80ff75ab03923e4 2024-11-10T09:28:18,915 INFO [M:0;802346a334f5:38235 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37473/user/jenkins/test-data/56ac3a83-a4d6-9047-baa9-4c1caf03bbd6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b4b6df6a202b43a7a80ff75ab03923e4, entries=1, sequenceid=2045, filesize=5.0 K 2024-11-10T09:28:18,916 INFO [M:0;802346a334f5:38235 {}] regionserver.HRegion(3140): Finished flush of dataSize ~732.96 KB/750554, heapSize ~898.95 KB/920520, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2045, compaction requested=false 2024-11-10T09:28:18,917 INFO [M:0;802346a334f5:38235 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T09:28:18,917 DEBUG [M:0;802346a334f5:38235 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731230897607Disabling compacts and flushes for region at 1731230897607Disabling writes for close at 1731230897608 (+1 ms)Obtaining lock to block concurrent updates at 1731230897608Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731230897608Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=750554, getHeapSize=920760, getOffHeapSize=0, getCellsCount=2238 at 1731230897609 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731230897610 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731230897610Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731230897628 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731230897628Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731230898040 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731230898059 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731230898059Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731230898475 (+416 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731230898490 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731230898490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c16f19a: reopening flushed file at 1731230898902 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@711c0973: reopening flushed file at 1731230898908 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4539ed09: reopening flushed file at 1731230898912 (+4 ms)Finished flush of dataSize ~732.96 KB/750554, heapSize ~898.95 KB/920520, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1308ms, sequenceid=2045, compaction requested=false at 1731230898916 (+4 ms)Writing region close event to WAL at 1731230898917 (+1 ms)Closed at 1731230898917 2024-11-10T09:28:18,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32877 is added to blk_1073741830_1006 (size=884670) 2024-11-10T09:28:18,920 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T09:28:18,921 INFO [M:0;802346a334f5:38235 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T09:28:18,921 INFO [M:0;802346a334f5:38235 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38235 2024-11-10T09:28:18,921 INFO [M:0;802346a334f5:38235 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T09:28:19,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:28:19,198 INFO [M:0;802346a334f5:38235 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T09:28:19,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38235-0x101241016650000, quorum=127.0.0.1:57057, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T09:28:19,209 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T09:28:19,211 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T09:28:19,211 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T09:28:19,211 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T09:28:19,211 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/hadoop.log.dir/,STOPPED} 2024-11-10T09:28:19,213 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T09:28:19,214 WARN [BP-1012386263-172.17.0.3-1731230698324 heartbeating to localhost/127.0.0.1:37473 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T09:28:19,214 WARN [BP-1012386263-172.17.0.3-1731230698324 heartbeating to localhost/127.0.0.1:37473 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1012386263-172.17.0.3-1731230698324 (Datanode Uuid 23b75091-1942-4c7e-b8ea-bd802bc427a3) service to localhost/127.0.0.1:37473 2024-11-10T09:28:19,214 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T09:28:19,216 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/data/data1/current/BP-1012386263-172.17.0.3-1731230698324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T09:28:19,216 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/cluster_ae6ee3eb-680b-2662-e81b-befc5ae54d1f/data/data2/current/BP-1012386263-172.17.0.3-1731230698324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T09:28:19,217 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T09:28:19,223 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T09:28:19,224 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T09:28:19,224 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T09:28:19,224 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T09:28:19,224 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c39215-c868-72fc-8519-c30b83d5c4d5/hadoop.log.dir/,STOPPED} 2024-11-10T09:28:19,238 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T09:28:19,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down